From 7f56f583d891859c94b24db0ec38a301c3f3466a Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Mon, 23 Feb 2015 14:59:51 -0500
Subject: [PATCH] High: yield to scheduler during new connection auth
 processing

---
 lib/ipc_setup.c | 233 ++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 159 insertions(+), 74 deletions(-)

diff --git a/lib/ipc_setup.c b/lib/ipc_setup.c
index 1ea085b..144b4c0 100644
--- a/lib/ipc_setup.c
+++ b/lib/ipc_setup.c
@@ -48,6 +48,24 @@ struct ipc_auth_ugp {
 	pid_t pid;
 };
 
+struct ipc_auth_data {
+	int32_t sock;
+	struct qb_ipcs_service *s;
+	struct qb_ipc_connection_request msg;
+	struct msghdr msg_recv;
+	struct iovec iov_recv;
+	struct ipc_auth_ugp ugp;
+
+	size_t processed;
+	size_t len;
+
+#ifdef SO_PASSCRED
+	char *cmsg_cred;
+#endif
+
+};
+
+
 static int32_t qb_ipcs_us_connection_acceptor(int fd, int revent, void *data);
 
 ssize_t
@@ -83,20 +101,21 @@ retry_send:
 }
 
 static ssize_t
-qb_ipc_us_recv_msghdr(int32_t s, struct msghdr *hdr, char *msg, size_t len)
+qb_ipc_us_recv_msghdr(struct ipc_auth_data *data)
 {
+	char *msg = (char *) &data->msg;
 	int32_t result;
-	int32_t processed = 0;
 
 	qb_sigpipe_ctl(QB_SIGPIPE_IGNORE);
 
 retry_recv:
-	hdr->msg_iov->iov_base = &msg[processed];
-	hdr->msg_iov->iov_len = len - processed;
+	data->msg_recv.msg_iov->iov_base = &msg[data->processed];
+	data->msg_recv.msg_iov->iov_len = data->len - data->processed;
 
-	result = recvmsg(s, hdr, MSG_NOSIGNAL | MSG_WAITALL);
+	result = recvmsg(data->sock, &data->msg_recv, MSG_NOSIGNAL | MSG_WAITALL);
 	if (result == -1 && errno == EAGAIN) {
-		goto retry_recv;
+		qb_sigpipe_ctl(QB_SIGPIPE_DEFAULT);
+		return -EAGAIN;
 	}
 	if (result == -1) {
 		qb_sigpipe_ctl(QB_SIGPIPE_DEFAULT);
@@ -105,18 +124,18 @@ retry_recv:
 	if (result == 0) {
 		qb_sigpipe_ctl(QB_SIGPIPE_DEFAULT);
 		qb_util_log(LOG_DEBUG,
-			    "recv(fd %d) got 0 bytes assuming ENOTCONN", s);
+			    "recv(fd %d) got 0 bytes assuming ENOTCONN", data->sock);
 		return -ENOTCONN;
 	}
 
-	processed += result;
-	if (processed != len) {
+	data->processed += result;
+	if (data->processed != data->len) {
 		goto retry_recv;
 	}
 	qb_sigpipe_ctl(QB_SIGPIPE_DEFAULT);
-	assert(processed == len);
+	assert(data->processed == data->len);
 
-	return processed;
+	return data->processed;
 }
 
 int32_t
@@ -434,6 +453,7 @@ qb_ipcs_us_withdraw(struct qb_ipcs_service * s)
 	(void)s->poll_fns.dispatch_del(s->server_sock);
 	shutdown(s->server_sock, SHUT_RDWR);
 	close(s->server_sock);
+	s->server_sock = -1;
 	return 0;
 }
 
@@ -541,45 +561,56 @@ send_response:
 	return res;
 }
 
-static int32_t
-qb_ipcs_uc_recv_and_auth(int32_t sock, void *msg, size_t len,
-			 struct ipc_auth_ugp *ugp)
+static void
+destroy_ipc_auth_data(struct ipc_auth_data *data)
 {
-	int32_t res = 0;
-	struct msghdr msg_recv;
-	struct iovec iov_recv;
+	if (data->s) {
+		qb_ipcs_unref(data->s);
+	}
 
 #ifdef SO_PASSCRED
-	char cmsg_cred[CMSG_SPACE(sizeof(struct ucred))];
-	int off = 0;
-	int on = 1;
-#endif
-	msg_recv.msg_iov = &iov_recv;
-	msg_recv.msg_iovlen = 1;
-	msg_recv.msg_name = 0;
-	msg_recv.msg_namelen = 0;
-#ifdef SO_PASSCRED
-	msg_recv.msg_control = (void *)cmsg_cred;
-	msg_recv.msg_controllen = sizeof(cmsg_cred);
+	free(data->cmsg_cred);
 #endif
-#ifdef QB_SOLARIS
-	msg_recv.msg_accrights = 0;
-	msg_recv.msg_accrightslen = 0;
-#else
-	msg_recv.msg_flags = 0;
-#endif /* QB_SOLARIS */
+	free(data);
+}
+
+static int32_t
+process_auth(int32_t fd, int32_t revents, void *d)
+{
+	struct ipc_auth_data *data = (struct ipc_auth_data *) d;
 
-	iov_recv.iov_base = msg;
-	iov_recv.iov_len = len;
+	int32_t res = 0;
 #ifdef SO_PASSCRED
-	setsockopt(sock, SOL_SOCKET, SO_PASSCRED, &on, sizeof(on));
+	int off = 0;
 #endif
 
-	res = qb_ipc_us_recv_msghdr(sock, &msg_recv, msg, len);
-	if (res < 0) {
+	if (data->s->server_sock == -1) {
+		qb_util_log(LOG_DEBUG, "Closing fd (%d) for server shutdown", fd);
+		res = -ESHUTDOWN;
 		goto cleanup_and_return;
 	}
-	if (res != len) {
+
+	if (revents & POLLNVAL) {
+		qb_util_log(LOG_DEBUG, "NVAL conn fd (%d)", fd);
+		res = -EINVAL;
+		goto cleanup_and_return;
+	}
+	if (revents & POLLHUP) {
+		qb_util_log(LOG_DEBUG, "HUP conn fd (%d)", fd);
+		res = -ESHUTDOWN;
+		goto cleanup_and_return;
+	}
+	if ((revents & POLLIN) == 0) {
+		return 0;
+	}
+
+	res = qb_ipc_us_recv_msghdr(data);
+	if (res == -EAGAIN) {
+		/* yield to mainloop, Let mainloop call us again */
+		return 0;
+	}
+
+	if (res != data->len) {
 		res = -EIO;
 		goto cleanup_and_return;
 	}
@@ -595,11 +626,11 @@ qb_ipcs_uc_recv_and_auth(int32_t sock, void *msg, size_t len,
 	{
 		ucred_t *uc = NULL;
 
-		if (getpeerucred(sock, &uc) == 0) {
+		if (getpeerucred(data->sock, &uc) == 0) {
 			res = 0;
-			ugp->uid = ucred_geteuid(uc);
-			ugp->gid = ucred_getegid(uc);
-			ugp->pid = ucred_getpid(uc);
+			ugp.uid = ucred_geteuid(uc);
+			ugp.gid = ucred_getegid(uc);
+			ugp.pid = ucred_getpid(uc);
 			ucred_free(uc);
 		} else {
 			res = -errno;
@@ -614,7 +645,7 @@ qb_ipcs_uc_recv_and_auth(int32_t sock, void *msg, size_t len,
 		 * TODO get the peer's pid.
 		 * c->pid = ?;
 		 */
-		if (getpeereid(sock, &ugp->uid, &ugp->gid) == 0) {
+		if (getpeereid(data->sock, &ugp.uid, &ugp.gid) == 0) {
 			res = 0;
 		} else {
 			res = -errno;
@@ -630,33 +661,105 @@ qb_ipcs_uc_recv_and_auth(int32_t sock, void *msg, size_t len,
 		struct cmsghdr *cmsg;
 
 		res = -EINVAL;
-		for (cmsg = CMSG_FIRSTHDR(&msg_recv); cmsg != NULL;
-		     cmsg = CMSG_NXTHDR(&msg_recv, cmsg)) {
+		for (cmsg = CMSG_FIRSTHDR(&data->msg_recv); cmsg != NULL;
+		     cmsg = CMSG_NXTHDR(&data->msg_recv, cmsg)) {
 			if (cmsg->cmsg_type != SCM_CREDENTIALS)
 				continue;
 
 			memcpy(&cred, CMSG_DATA(cmsg), sizeof(struct ucred));
 			res = 0;
-			ugp->pid = cred.pid;
-			ugp->uid = cred.uid;
-			ugp->gid = cred.gid;
+			data->ugp.pid = cred.pid;
+			data->ugp.uid = cred.uid;
+			data->ugp.gid = cred.gid;
 			break;
 		}
 	}
 #else /* no credentials */
-	ugp->pid = 0;
-	ugp->uid = 0;
-	ugp->gid = 0;
+	data->ugp.pid = 0;
+	data->ugp.uid = 0;
+	data->ugp.gid = 0;
 	res = -ENOTSUP;
 #endif /* no credentials */
 
 cleanup_and_return:
+#ifdef SO_PASSCRED
+	setsockopt(data->sock, SOL_SOCKET, SO_PASSCRED, &off, sizeof(off));
+#endif
 
+	(void)data->s->poll_fns.dispatch_del(data->sock);
+
+	if (res < 0) {
+		close(data->sock);
+	} else if (data->msg.hdr.id == QB_IPC_MSG_AUTHENTICATE) {
+		(void)handle_new_connection(data->s, res, data->sock, &data->msg, data->len, &data->ugp);
+	} else {
+		close(data->sock);
+	}
+	destroy_ipc_auth_data(data);
+
+	return 1;
+}
+
+static void
+qb_ipcs_uc_recv_and_auth(int32_t sock, struct qb_ipcs_service *s)
+{
+	int res = 0;
+	struct ipc_auth_data *data = NULL;
 #ifdef SO_PASSCRED
-	setsockopt(sock, SOL_SOCKET, SO_PASSCRED, &off, sizeof(off));
+	int on = 1;
 #endif
 
-	return res;
+	data = calloc(1, sizeof(struct ipc_auth_data));
+	if (data == NULL) {
+		close(sock);
+		/* -ENOMEM */
+		return;
+	}
+
+	data->s = s;
+	qb_ipcs_ref(data->s);
+
+	data->msg_recv.msg_iov = &data->iov_recv;
+	data->msg_recv.msg_iovlen = 1;
+	data->msg_recv.msg_name = 0;
+	data->msg_recv.msg_namelen = 0;
+
+#ifdef SO_PASSCRED
+	data->cmsg_cred = calloc(1,CMSG_SPACE(sizeof(struct ucred)));
+	if (data->cmsg_cred == NULL) {
+		close(sock);
+		destroy_ipc_auth_data(data);
+		/* -ENOMEM */
+		return;
+	}
+	data->msg_recv.msg_control = (void *)data->cmsg_cred;
+	data->msg_recv.msg_controllen = CMSG_SPACE(sizeof(struct ucred));
+#endif
+#ifdef QB_SOLARIS
+	data->msg_recv.msg_accrights = 0;
+	data->msg_recv.msg_accrightslen = 0;
+#else
+	data->msg_recv.msg_flags = 0;
+#endif /* QB_SOLARIS */
+
+	data->len = sizeof(struct qb_ipc_connection_request);
+	data->iov_recv.iov_base = &data->msg;
+	data->iov_recv.iov_len = data->len;
+	data->sock = sock;
+
+#ifdef SO_PASSCRED
+	setsockopt(sock, SOL_SOCKET, SO_PASSCRED, &on, sizeof(on));
+#endif
+
+	res = s->poll_fns.dispatch_add(QB_LOOP_MED,
+					data->sock,
+					POLLIN | POLLPRI | POLLNVAL,
+					data, process_auth);
+	if (res < 0) {
+		qb_util_log(LOG_DEBUG, "Failed to process AUTH for fd (%d)", data->sock);
+		close(sock);
+		destroy_ipc_auth_data(data);
+	}
 }
 
 static int32_t
@@ -666,8 +769,6 @@ qb_ipcs_us_connection_acceptor(int fd, int revent, void *data)
 	int32_t new_fd;
 	struct qb_ipcs_service *s = (struct qb_ipcs_service *)data;
 	int32_t res;
-	struct qb_ipc_connection_request setup_msg;
-	struct ipc_auth_ugp ugp;
 	socklen_t addrlen = sizeof(struct sockaddr_un);
 
 	if (revent & (POLLNVAL | POLLHUP | POLLERR)) {
@@ -707,22 +808,6 @@ retry_accept:
 		return 0;
 	}
 
-	res = qb_ipcs_uc_recv_and_auth(new_fd, &setup_msg, sizeof(setup_msg),
-				       &ugp);
-	if (res < 0) {
-		close(new_fd);
-		/* This is an error, but -1 would indicate disconnect
-		 * from the poll loop
-		 */
-		return 0;
-	}
-
-	if (setup_msg.hdr.id == QB_IPC_MSG_AUTHENTICATE) {
-		(void)handle_new_connection(s, res, new_fd, &setup_msg,
-					    sizeof(setup_msg), &ugp);
-	} else {
-		close(new_fd);
-	}
-
+	qb_ipcs_uc_recv_and_auth(new_fd, s);
 	return 0;
 }
-- 
1.8.4.2