Try reconnecting if rpcbind EPIPEs us

If the NIS server encounters an EPIPE while attempting to communicate
with the portmapper, try to reconnect before giving up on registering.
Depending on which RPC implementation is used, rpcbind may drop idle
clients after 30 seconds, and our startup can take longer than that.

3 files changed, 94 insertions, 60 deletions

diff --git a/src/plug-nis.c b/src/plug-nis.c
index a3aa8c1..a1bdf89 100644
--- a/src/plug-nis.c
+++ b/src/plug-nis.c
@@ -102,19 +102,19 @@ plugin_startup(Slapi_PBlock *pb)
 	if (state->pmap_client_socket != -1) {
 		/* Try to kick off any other NIS servers on the local box. */
 		portmap_unregister(plugin_description.spd_id,
-				   state->pmap_client_socket,
+				   &state->pmap_client_socket, 0,
 				   YPPROG, YPVERS,
 				   AF_INET6, IPPROTO_TCP, 0);
 		portmap_unregister(plugin_description.spd_id,
-				   state->pmap_client_socket,
+				   &state->pmap_client_socket, 0,
 				   YPPROG, YPVERS,
 				   AF_INET6, IPPROTO_UDP, 0);
 		portmap_unregister(plugin_description.spd_id,
-				   state->pmap_client_socket,
+				   &state->pmap_client_socket, 0,
 				   YPPROG, YPVERS,
 				   AF_INET, IPPROTO_TCP, 0);
 		portmap_unregister(plugin_description.spd_id,
-				   state->pmap_client_socket,
+				   &state->pmap_client_socket, 0,
 				   YPPROG, YPVERS,
 				   AF_INET, IPPROTO_UDP, 0);
 		/* Register our listening ports. */
@@ -134,7 +134,8 @@ plugin_startup(Slapi_PBlock *pb)
 				break;
 			}
 			if (!portmap_register(plugin_description.spd_id,
-					      state->pmap_client_socket,
+					      &state->pmap_client_socket,
+					      state->listener[i].port,
 					      YPPROG, YPVERS,
 					      state->listener[i].pf,
 					      protocol,
@@ -153,7 +154,8 @@ plugin_startup(Slapi_PBlock *pb)
 				 * register for them, too. */
 				if (state->listener[i].pf == AF_INET6) {
 					portmap_register(plugin_description.spd_id,
-							 state->pmap_client_socket,
+							 &state->pmap_client_socket,
+							 state->listener[i].port,
 							 YPPROG, YPVERS,
 							 AF_INET,
 							 protocol,
@@ -190,14 +192,16 @@ plugin_shutdown(Slapi_PBlock *pb)
 				break;
 			}
 			portmap_unregister(plugin_description.spd_id,
-					   state->pmap_client_socket,
+					   &state->pmap_client_socket,
+					   state->listener[i].port,
 					   YPPROG, YPVERS,
 					   state->listener[i].pf,
 					   protocol,
 					   state->listener[i].port);
 			if (state->listener[i].pf == AF_INET6) {
 				portmap_unregister(plugin_description.spd_id,
-						   state->pmap_client_socket,
+						   &state->pmap_client_socket,
+						   state->listener[i].port,
 						   YPPROG, YPVERS,
 						   AF_INET,
 						   protocol,
diff --git a/src/portmap.c b/src/portmap.c
index 5b463c0..f2e009e 100644
--- a/src/portmap.c
+++ b/src/portmap.c
@@ -96,38 +96,39 @@ main(int argc, char **argv)
 	}
 	setregid(2516, 2516);
 	setreuid(2510, 2510);
-	portmap_unregister("portmap", s, YPPROG, YPVERS, AF_INET, IPPROTO_TCP,
-			   0);
-	portmap_unregister("portmap", s, YPPROG, YPVERS, AF_INET, IPPROTO_UDP,
-			   0);
-	portmap_unregister("portmap", s, YPPROG, YPVERS, AF_INET6, IPPROTO_TCP,
-			   0);
-	portmap_unregister("portmap", s, YPPROG, YPVERS, AF_INET6, IPPROTO_UDP,
-			   0);
-	portmap_register("portmap", s, YPPROG, YPVERS, AF_INET, IPPROTO_TCP,
-			 port);
-	portmap_register("portmap", s, YPPROG, YPVERS, AF_INET, IPPROTO_UDP,
-			 port);
-	portmap_register("portmap", s, YPPROG, YPVERS, AF_INET6, IPPROTO_TCP,
-			 port);
-	portmap_register("portmap", s, YPPROG, YPVERS, AF_INET6, IPPROTO_UDP,
-			 port);
+	sleep(60);
+	portmap_unregister("portmap", &s, port, YPPROG, YPVERS,
+			   AF_INET, IPPROTO_TCP, 0);
+	portmap_unregister("portmap", &s, port, YPPROG, YPVERS,
+			   AF_INET, IPPROTO_UDP, 0);
+	portmap_unregister("portmap", &s, port, YPPROG, YPVERS,
+			   AF_INET6, IPPROTO_TCP, 0);
+	portmap_unregister("portmap", &s, port, YPPROG, YPVERS,
+			   AF_INET6, IPPROTO_UDP, 0);
+	portmap_register("portmap", &s, port, YPPROG, YPVERS,
+			 AF_INET, IPPROTO_TCP, port);
+	portmap_register("portmap", &s, port, YPPROG, YPVERS,
+			 AF_INET, IPPROTO_UDP, port);
+	portmap_register("portmap", &s, port, YPPROG, YPVERS,
+			 AF_INET6, IPPROTO_TCP, port);
+	portmap_register("portmap", &s, port, YPPROG, YPVERS,
+			 AF_INET6, IPPROTO_UDP, port);
 	ret = system("rpcinfo | grep ypserv");
 	ret = system("rpcinfo -p | grep ypserv");
-	portmap_unregister("portmap", s, YPPROG, YPVERS, AF_INET, IPPROTO_TCP,
-			   port);
-	portmap_unregister("portmap", s, YPPROG, YPVERS, AF_INET, IPPROTO_UDP,
-			   port);
-	portmap_unregister("portmap", s, YPPROG, YPVERS, AF_INET6, IPPROTO_TCP,
-			   port);
-	portmap_unregister("portmap", s, YPPROG, YPVERS, AF_INET6, IPPROTO_UDP,
-			   port);
+	portmap_unregister("portmap", &s, port, YPPROG, YPVERS,
+			   AF_INET, IPPROTO_TCP, port);
+	portmap_unregister("portmap", &s, port, YPPROG, YPVERS,
+			   AF_INET, IPPROTO_UDP, port);
+	portmap_unregister("portmap", &s, port, YPPROG, YPVERS,
+			   AF_INET6, IPPROTO_TCP, port);
+	portmap_unregister("portmap", &s, port, YPPROG, YPVERS,
+			   AF_INET6, IPPROTO_UDP, port);
 	return 0;
 }
 #endif
 
 static bool_t
-portmap_register_work(const char *module, int client_sock,
+portmap_register_work(const char *module, int *client_sock, int but_not,
 		      bool_t stream,
 		      struct sockaddr *dgram_address, socklen_t addrlen,
 		      int prog, int vers, int proc,
@@ -142,7 +143,7 @@ portmap_register_work(const char *module, int client_sock,
 	bool_t ret = FALSE;
 	struct sockaddr addr;
 	struct pollfd pollfd;
-	int i;
+	int i, err, sock2 = -1;
 	static u_long xid;
 
 	log_id = (char *) module;
@@ -197,7 +198,7 @@ portmap_register_work(const char *module, int client_sock,
 		portmap_length += 4;
 	} else {
 		/* Point the datagram socket at the remote. */
-		if (connect(client_sock, dgram_address, addrlen) != 0) {
+		if (connect(*client_sock, dgram_address, addrlen) != 0) {
 			slapi_log_error(SLAPI_LOG_FATAL, log_id,
 					"error targeting portmap: %s\n",
 					strerror(errno));
@@ -210,26 +211,56 @@ portmap_register_work(const char *module, int client_sock,
 	for (i = 1; i < 32; i *= 2) {
 		/* Try to send our request.  If there's any problem,
 		 * immediately retry. */
-		if (send(client_sock, &portmap_buf, portmap_length,
+		if (send(*client_sock, &portmap_buf, portmap_length,
 			 MSG_NOSIGNAL) != portmap_length) {
+			err = errno;
 			slapi_log_error(SLAPI_LOG_FATAL, log_id,
 					"error sending request to portmap or "
-					"rpcbind on %d: %s\n", client_sock,
-					strerror(errno));
+					"rpcbind on %d: %s\n", *client_sock,
+					strerror(err));
 			if (stream) {
-				break;
+				if (err == EPIPE) {
+					/* Try again with a new client
+					 * connection -- some RPC
+					 * implementations will cause rpcbind
+					 * to drop idle clients. */
+					snprintf(reply_buf, sizeof(reply_buf),
+						 "%s", module);
+					sock2 = portmap_create_client_socket(reply_buf,
+									     but_not);
+					if (sock2 != -1) {
+						if (send(sock2, &portmap_buf, portmap_length,
+							 MSG_NOSIGNAL) == portmap_length) {
+							slapi_log_error(SLAPI_LOG_FATAL, log_id,
+									"retried sending request "
+									"to portmap or rpcbind "
+									"on %d, and succeeded\n",
+									sock2);
+							close(*client_sock);
+							*client_sock = sock2;
+						} else {
+							/* Still got an error -- bail. */
+							close(sock2);
+							break;
+						}
+					} else {
+						break;
+					}
+				} else {
+					break;
+				}
 			} else {
 				continue;
 			}
 		}
 
 		/* Wait for a response. */
-		pollfd.fd = client_sock;
+		pollfd.fd = *client_sock;
 		pollfd.events = POLLIN | POLLERR;
 		if ((poll(&pollfd, 1, stream ? -1 : i * 1000) > 0) &&
 		    (pollfd.revents & POLLIN)) {
 			/* Read the response. */
-			reply_length = recv(client_sock,
+			reply_length = recv(*client_sock,
 					    reply_buf, sizeof(reply_buf), 0);
 			/* Decode the response. */
 			if (reply_length > 0) {
@@ -266,6 +297,8 @@ portmap_register_work(const char *module, int client_sock,
 					if ((msg.rm_direction == REPLY) &&
 					    (msg.rm_xid == xid)) {
 						xdr_destroy(&portmap_xdrs);
+						memset(&portmap_xdrs, 0,
+						       sizeof(portmap_xdrs));
 						break;
 					}
 				}
@@ -280,7 +313,7 @@ portmap_register_work(const char *module, int client_sock,
 		 * socket around. */
 		memset(&addr, 0, sizeof(addr));
 		addr.sa_family = AF_UNSPEC;
-		connect(client_sock, &addr, sizeof(addr));
+		connect(*client_sock, &addr, sizeof(addr));
 		/* Check for a timeout. */
 		if (i == 32) {
 			slapi_log_error(SLAPI_LOG_FATAL, log_id,
@@ -336,7 +369,6 @@ portmap_register_work(const char *module, int client_sock,
 					"RPC mismatch\n");
 			break;
 		}
-		xdr_destroy(&portmap_xdrs);
 		return FALSE;
 	}
 
@@ -355,7 +387,6 @@ portmap_register_work(const char *module, int client_sock,
 	if (msg.rm_reply.rp_acpt.ar_stat != SUCCESS) {
 		slapi_log_error(SLAPI_LOG_FATAL, log_id,
 				"portmap request not processed\n");
-		xdr_destroy(&portmap_xdrs);
 		return FALSE;
 	}
 
@@ -368,8 +399,6 @@ portmap_register_work(const char *module, int client_sock,
 				"portmap request failed\n");
 	}
 
-	xdr_destroy(&portmap_xdrs);
-
 	return ret;
 }
 
@@ -392,7 +421,7 @@ portmap_xdr_rpcbind_req(XDR *xdrs, struct rpcbind_req *req)
 }
 
 static bool_t
-portmap_register_rpcbind(const char *module, int client_sock,
+portmap_register_rpcbind(const char *module, int *client_sock, int but_not,
 			 bool_t create,
 		         int family, int protocol, int port,
 			 int program, int version)
@@ -444,7 +473,8 @@ portmap_register_rpcbind(const char *module, int client_sock,
 		req.address = NULL;
 		break;
 	}
-	return portmap_register_work(module, client_sock, TRUE, NULL, 0,
+	return portmap_register_work(module, client_sock, but_not,
+				     TRUE, NULL, 0,
 				     RPCBPROG, RPCBVERS,
 				     create ? RPCBPROC_SET : RPCBPROC_UNSET,
 				     &req,
@@ -452,7 +482,7 @@ portmap_register_rpcbind(const char *module, int client_sock,
 }
 
 static bool_t
-portmap_register_portmap(const char *module, int client_sock,
+portmap_register_portmap(const char *module, int *client_sock, int but_not,
 			 bool_t create,
 		         int protocol, int port, int program, int version)
 {
@@ -466,7 +496,7 @@ portmap_register_portmap(const char *module, int client_sock,
 	map.pm_vers = version;
 	map.pm_prot = protocol;
 	map.pm_port = port;
-	return portmap_register_work(module, client_sock,
+	return portmap_register_work(module, client_sock, but_not,
 				     FALSE,
 				     (struct sockaddr *) &sin, sizeof(sin),
 				     PMAPPROG, PMAPVERS,
@@ -486,32 +516,32 @@ portmap_is_stream(int sd)
 }
 
 bool_t
-portmap_register(const char *log_id, int resv_sock,
+portmap_register(const char *log_id, int *resv_sock, int but_not,
 		 int program, int version,
 		 int family, int protocol, int port)
 {
-	return portmap_is_stream(resv_sock) ?
-	       portmap_register_rpcbind(log_id, resv_sock,
+	return portmap_is_stream(*resv_sock) ?
+	       portmap_register_rpcbind(log_id, resv_sock, but_not,
 					TRUE,
 					family, protocol, port,
 					program, version) :
-	       portmap_register_portmap(log_id, resv_sock,
+	       portmap_register_portmap(log_id, resv_sock, but_not,
 					TRUE,
 					protocol, port,
 					program, version);
 }
 
 bool_t
-portmap_unregister(const char *log_id, int resv_sock,
+portmap_unregister(const char *log_id, int *resv_sock, int but_not,
 		   int program, int version,
 		   int family, int protocol, int port)
 {
-	return portmap_is_stream(resv_sock) ?
-	       portmap_register_rpcbind(log_id, resv_sock,
+	return portmap_is_stream(*resv_sock) ?
+	       portmap_register_rpcbind(log_id, resv_sock, but_not,
 					FALSE,
 					family, protocol, port,
 					program, version) :
-	       portmap_register_portmap(log_id, resv_sock,
+	       portmap_register_portmap(log_id, resv_sock, but_not,
 					FALSE,
 					protocol, port,
 					program, version);
diff --git a/src/portmap.h b/src/portmap.h
index 7cbb56c..0f302fc 100644
--- a/src/portmap.h
+++ b/src/portmap.h
@@ -22,10 +22,10 @@
 #ifndef portmap_h
 #define portmap_h
 
-bool_t portmap_register(const char *log_id, int resv_sock,
+bool_t portmap_register(const char *log_id, int *resv_sock, int but_not,
 			int program, int version,
 			int family, int protocol, int port);
-bool_t portmap_unregister(const char *log_id, int resv_sock,
+bool_t portmap_unregister(const char *log_id, int *resv_sock, int but_not,
 			  int program, int version,
 			  int family, int protocol, int port);
 int portmap_create_client_socket(char *module, int but_not);