protocol/server: make listen backlog value as configurable

problem:

When we call listen from protocol/server, we are giving a
hard coded valie of 10 if it is not manually given.
With multiplexing, especially when glusterd restarts all
clients may try to connect to the server at a time.
Which will result in overflowing the queue, and kernel
will complain about the errors.

Solution:

This patch will introduce a volume set command to make backlog
value as a configurable. This patch also changes the default
values for backlog from 10 to 128. This changes is only applicable
for sockets listening from protocol.

Example:

gluster volume set <volname> transport.listen-backlog 1024

Note: 1 Brick has to be restarted to get this value in effect
      2 This changes won't be reflected in glusterd, or other
        xlators which calls listen. If you need, you have to
        add this option to the volfile.

Change-Id: I0c5a2bbf28b5db612f9979e7560e05dd82b41477
BUG: 1456405
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
Reviewed-on: https://review.gluster.org/17411
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-by: Raghavendra Talur <rtalur@redhat.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Reviewed-by: Niels de Vos <ndevos@redhat.com>
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>

11 files changed, 124 insertions, 18 deletions

diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
index 4c668dcb19..a23c45e4a6 100644
--- a/libglusterfs/src/globals.h
+++ b/libglusterfs/src/globals.h
@@ -84,6 +84,8 @@
 
 #define GD_OP_VERSION_3_11_0   31100 /* Op-version for GlusterFS 3.11.0 */
 
+#define GD_OP_VERSION_3_11_1   31101 /* Op-version for GlusterFS 3.11.1 */
+
 #define GD_OP_VERSION_4_0_0    40000 /* Op-version for GlusterFS 4.0.0 */
 
 #define GD_OP_VER_PERSISTENT_AFR_XATTRS GD_OP_VERSION_3_6_0
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 6feefb85e9..aec1e029df 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -72,6 +72,7 @@
 #endif
 
 #define GLUSTERD_MAX_SNAP_NAME  255
+#define GLUSTERFS_SOCKET_LISTEN_BACKLOG  10
 #define ZR_MOUNTPOINT_OPT       "mountpoint"
 #define ZR_ATTR_TIMEOUT_OPT     "attribute-timeout"
 #define ZR_ENTRY_TIMEOUT_OPT    "entry-timeout"
diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c
index 8d9e6474f3..01f96c21b5 100644
--- a/rpc/rpc-transport/rdma/src/rdma.c
+++ b/rpc/rpc-transport/rdma/src/rdma.c
@@ -4499,6 +4499,12 @@ gf_rdma_options_init (rpc_transport_t *this)
         options->attr_retry_cnt = GF_RDMA_RETRY_CNT;
         options->attr_rnr_retry = GF_RDMA_RNR_RETRY;
 
+        temp = dict_get (this->options, "transport.listen-backlog");
+        if (temp)
+                options->backlog = data_to_uint32 (temp);
+        else
+                options->backlog = GLUSTERFS_SOCKET_LISTEN_BACKLOG;
+
         temp = dict_get (this->options,
                          "transport.rdma.work-request-send-count");
         if (temp)
@@ -4635,6 +4641,7 @@ gf_rdma_init (rpc_transport_t *this)
         priv->peer.recv_count = options->recv_count;
         priv->peer.send_size = options->send_size;
         priv->peer.recv_size = options->recv_size;
+        priv->backlog = options->backlog;
 
         priv->peer.trans = this;
         INIT_LIST_HEAD (&priv->peer.ioq);
@@ -4848,7 +4855,8 @@ gf_rdma_listen (rpc_transport_t *this)
                 goto err;
         }
 
-        ret = rdma_listen (peer->cm_id, 10);
+        ret = rdma_listen (peer->cm_id, priv->backlog);
+
         if (ret != 0) {
                 gf_msg (this->name, GF_LOG_WARNING, errno,
                         RDMA_MSG_LISTEN_FAILED,
@@ -4919,6 +4927,28 @@ init (rpc_transport_t *this)
         return 0;
 }
 
+int
+reconfigure (rpc_transport_t *this, dict_t *options)
+{
+        gf_rdma_private_t *priv          = NULL;
+        uint32_t          backlog        = 0;
+        int               ret            = -1;
+
+        GF_VALIDATE_OR_GOTO ("rdma", this, out);
+        GF_VALIDATE_OR_GOTO ("rdma", this->private, out);
+
+        priv = this->private;
+
+        if (dict_get_uint32 (options, "transport.listen-backlog",
+                             &backlog) == 0) {
+                priv->backlog = backlog;
+                gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+                        "transport.listen-backlog=%d", priv->backlog);
+        }
+        ret = 0;
+out:
+        return ret;
+}
 void
 fini (struct rpc_transport *this)
 {
diff --git a/rpc/rpc-transport/rdma/src/rdma.h b/rpc/rpc-transport/rdma/src/rdma.h
index 449861f075..6bcf6bc6ef 100644
--- a/rpc/rpc-transport/rdma/src/rdma.h
+++ b/rpc/rpc-transport/rdma/src/rdma.h
@@ -144,6 +144,7 @@ struct __gf_rdma_options {
 	uint8_t  attr_timeout;
 	uint8_t  attr_retry_cnt;
 	uint8_t  attr_rnr_retry;
+        uint32_t backlog;
 };
 typedef struct __gf_rdma_options gf_rdma_options_t;
 
@@ -383,6 +384,7 @@ struct __gf_rdma_private {
         pthread_mutex_t             recv_mutex;
         pthread_cond_t              recv_cond;
         gf_rdma_transport_entity_t  entity;
+        uint32_t                    backlog;
 };
 typedef struct __gf_rdma_private    gf_rdma_private_t;
 
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index f5062fb026..dd7f5d3b77 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -3536,10 +3536,7 @@ socket_listen (rpc_transport_t *this)
                         goto unlock;
                 }
 
-                if (priv->backlog)
-                        ret = listen (priv->sock, priv->backlog);
-                else
-                        ret = listen (priv->sock, 10);
+                ret = listen (priv->sock, priv->backlog);
 
                 if (ret == -1) {
                         gf_log (this->name, GF_LOG_ERROR,
@@ -3853,6 +3850,7 @@ reconfigure (rpc_transport_t *this, dict_t *options)
         gf_boolean_t      tmp_bool      = _gf_false;
         char             *optstr        = NULL;
         int               ret           = 0;
+        uint32_t          backlog       = 0;
         uint64_t          windowsize    = 0;
         uint32_t          timeout       = 0;
         int               keepaliveidle  = GF_KEEPALIVE_TIME;
@@ -3892,6 +3890,13 @@ reconfigure (rpc_transport_t *this, dict_t *options)
         gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
                 "transport.tcp-user-timeout=%d", priv->timeout);
 
+        if (dict_get_uint32 (options, "transport.listen-backlog",
+                             &backlog) == 0) {
+                priv->backlog = backlog;
+                gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+                        "transport.listen-backlog=%d", priv->backlog);
+        }
+
         if (dict_get_int32 (options, "transport.socket.keepalive-time",
                             &(priv->keepaliveidle)) != 0)
                 priv->keepaliveidle = keepaliveidle;
@@ -4190,10 +4195,12 @@ socket_init (rpc_transport_t *this)
                 "transport.keepalivecnt=%d", keepalivecnt);
 
         if (dict_get_uint32 (this->options,
-                             "transport.socket.listen-backlog",
-                             &backlog) == 0) {
-                priv->backlog = backlog;
+                             "transport.listen-backlog",
+                             &backlog) != 0) {
+
+                backlog = GLUSTERFS_SOCKET_LISTEN_BACKLOG;
         }
+        priv->backlog = backlog;
 
         optstr = NULL;
 
@@ -4612,9 +4619,6 @@ struct volume_options options[] = {
           .type  = GF_OPTION_TYPE_INT,
           .default_value = "9"
         },
-        { .key   = {"transport.socket.listen-backlog"},
-          .type  = GF_OPTION_TYPE_INT
-        },
         { .key   = {"transport.socket.read-fail-log"},
           .type  = GF_OPTION_TYPE_BOOL
         },
diff --git a/tests/basic/multiplex.t b/tests/basic/multiplex.t
index cfac357b2f..0448e5b12c 100644
--- a/tests/basic/multiplex.t
+++ b/tests/basic/multiplex.t
@@ -31,6 +31,8 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_up_bricks
 EXPECT 1 count_brick_processes
 
 TEST $CLI volume stop $V0
+#Testing the volume set command introduced for protocol/server
+TEST $CLI volume set $V0 transport.listen-backlog 1024
 EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT 0 count_brick_processes
 TEST $CLI volume start $V0
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_up_bricks
diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h
index e66107c3b2..9161d9058f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-messages.h
+++ b/xlators/mgmt/glusterd/src/glusterd-messages.h
@@ -41,7 +41,7 @@
 
 #define GLUSTERD_COMP_BASE      GLFS_MSGID_GLUSTERD
 
-#define GLFS_NUM_MESSAGES       601
+#define GLFS_NUM_MESSAGES       602
 
 #define GLFS_MSGID_END          (GLUSTERD_COMP_BASE + GLFS_NUM_MESSAGES + 1)
 /* Messaged with message IDs */
@@ -4861,6 +4861,14 @@
  */
 #define GD_MSG_PIDFILE_UNLINKING                   (GLUSTERD_COMP_BASE + 601)
 
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+#define GD_MSG_VOL_SET_VALIDATION_INFO             (GLUSTERD_COMP_BASE + 602)
+
 /*------------*/
 
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index d42a0ce47e..900d4be206 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -715,6 +715,52 @@ out:
 }
 
 static int
+validate_server_options (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                         char *value, char **op_errstr)
+{
+        char                 errstr[2048]  = "";
+        xlator_t            *this          = NULL;
+        int                  ret           = -1;
+        int                  origin_val    = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+                gf_msg (this->name, GF_LOG_INFO, 0,
+                        GD_MSG_VOL_SET_VALIDATION_INFO, "Please note that "
+                        "volume %s is started. This option will only get "
+                        "effected after a brick restart.", volinfo->volname);
+        }
+
+        ret = gf_string2int (value, &origin_val);
+        if (ret) {
+                snprintf (errstr, sizeof (errstr), "%s is not a compatible "
+                          "value. %s expects an integer value.", value, key);
+                ret = -1;
+                goto out;
+        }
+
+        if (origin_val < 0) {
+                snprintf (errstr, sizeof (errstr), "%s is not a "
+                          "compatible value. %s expects a positive"
+                          "integer value.", value, key);
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
+                        GD_MSG_INCOMPATIBLE_VALUE, "%s", errstr);
+                *op_errstr = gf_strdup (errstr);
+        }
+
+        return ret;
+}
+
+static int
 validate_stripe (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
                  char *value, char **op_errstr)
 {
@@ -1969,6 +2015,16 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .op_version  = GD_OP_VERSION_3_10_2,
           .value       = "9",
         },
+        { .key         = "transport.listen-backlog",
+          .voltype     = "protocol/server",
+          .option      = "transport.listen-backlog",
+          .op_version  = GD_OP_VERSION_3_11_1,
+          .validate_fn = validate_server_options,
+          .description = "This option uses the value of backlog argument that "
+                         "defines the maximum length to which the queue of "
+                         "pending connections for socket fd may grow.",
+          .value       = "10",
+        },
 
         /* Generic transport options */
         { .key         = SSL_OWN_CERT_OPT,
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 789bc6e9ed..14c1c6ae94 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -462,14 +462,12 @@ glusterd_rpcsvc_options_build (dict_t *options)
         int             ret = 0;
         uint32_t        backlog = 0;
 
-        ret = dict_get_uint32 (options, "transport.socket.listen-backlog",
-                               &backlog);
+        ret = dict_get_uint32 (options, "transport.listen-backlog", &backlog);
 
         if (ret) {
-                backlog = GLUSTERD_SOCKET_LISTEN_BACKLOG;
+                backlog = GLUSTERFS_SOCKET_LISTEN_BACKLOG;
                 ret = dict_set_uint32 (options,
-                                      "transport.socket.listen-backlog",
-                                      backlog);
+                                      "transport.listen-backlog", backlog);
                 if (ret)
                         goto out;
         }
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 1f7f47e443..9546a38990 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -40,7 +40,6 @@
 #include "events.h"
 
 #define GLUSTERD_TR_LOG_SIZE            50
-#define GLUSTERD_SOCKET_LISTEN_BACKLOG  128
 #define GLUSTERD_QUORUM_TYPE_KEY        "cluster.server-quorum-type"
 #define GLUSTERD_QUORUM_RATIO_KEY       "cluster.server-quorum-ratio"
 #define GLUSTERD_GLOBAL_OPT_VERSION     "global-option-version"
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index 130a55372a..4d89fe89b2 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -1606,6 +1606,10 @@ struct volume_options options[] = {
                     "socket*([ \t]),*([ \t])rdma"},
           .type  = GF_OPTION_TYPE_STR
         },
+        { .key   = {"transport.listen-backlog"},
+          .type  = GF_OPTION_TYPE_INT,
+          .default_value = "10",
+        },
         { .key   = {"volume-filename.*"},
           .type  = GF_OPTION_TYPE_PATH,
         },