From b78e56b839cb48bd923e07f286977e1b09fd6609 Mon Sep 17 00:00:00 2001

From: Mohammed Rafi KC <rkavunga@redhat.com>

Date: Tue, 13 Jun 2017 12:58:31 +0530

Subject: [PATCH 507/509] protocol/server: make listen backlog value as

 configurable

problem:

When we call listen from protocol/server, we are giving a

hard coded valie of 10 if it is not manually given.

With multiplexing, especially when glusterd restarts all

clients may try to connect to the server at a time.

Which will result in overflowing the queue, and kernel

will complain about the errors.

Solution:

This patch will introduce a volume set command to make backlog

value as a configurable. This patch also changes the default

values for backlog from 10 to 128. This changes is only applicable

for sockets listening from protocol.

Example:

gluster volume set <volname> transport.listen-backlog 1024

Note: 1 Brick has to be restarted to get this value in effect

      2 This changes won't be reflected in glusterd, or other

        xlators which calls listen. If you need, you have to

        add this option to the volfile.

Backport of>

>Change-Id: I0c5a2bbf28b5db612f9979e7560e05dd82b41477

>BUG: 1456405

>Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>

>Reviewed-on: https://review.gluster.org/17411

>Smoke: Gluster Build System <jenkins@build.gluster.org>

>NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

>CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

>Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

>Reviewed-by: Raghavendra Talur <rtalur@redhat.com>

>Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

>Reviewed-by: Niels de Vos <ndevos@redhat.com>

>Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>

Change-Id: I0c5a2bbf28b5db612f9979e7560e05dd82b41477

BUG: 1453145

Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/108900

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

---

 libglusterfs/src/globals.h                      |  2 +

 libglusterfs/src/glusterfs.h                    |  1 +

 rpc/rpc-transport/rdma/src/rdma.c               | 32 +++++++++++++-

 rpc/rpc-transport/rdma/src/rdma.h               |  2 +

 rpc/rpc-transport/socket/src/socket.c           | 25 ++++++-----

 tests/basic/multiplex.t                         |  3 ++

 xlators/mgmt/glusterd/src/glusterd-messages.h   | 10 ++++-

 xlators/mgmt/glusterd/src/glusterd-volume-set.c | 56 +++++++++++++++++++++++++

 xlators/mgmt/glusterd/src/glusterd.c            |  8 ++--

 xlators/mgmt/glusterd/src/glusterd.h            |  1 -

 xlators/protocol/server/src/server.c            |  4 ++

 11 files changed, 126 insertions(+), 18 deletions(-)

diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h

index 08031a7..c7a5c16 100644

--- a/libglusterfs/src/globals.h

+++ b/libglusterfs/src/globals.h

@@ -91,6 +91,8 @@

 #define GD_OP_VERSION_3_11_0   31100 /* Op-version for GlusterFS 3.11.0 */

+#define GD_OP_VERSION_3_11_1   31101 /* Op-version for GlusterFS 3.11.1 */

+

 #include "xlator.h"

 /* THIS */

diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h

index 3544aab..86519b0 100644

--- a/libglusterfs/src/glusterfs.h

+++ b/libglusterfs/src/glusterfs.h

@@ -71,6 +71,7 @@

 #endif

 #define GLUSTERD_MAX_SNAP_NAME  255

+#define GLUSTERFS_SOCKET_LISTEN_BACKLOG  10

 #define ZR_MOUNTPOINT_OPT       "mountpoint"

 #define ZR_ATTR_TIMEOUT_OPT     "attribute-timeout"

 #define ZR_ENTRY_TIMEOUT_OPT    "entry-timeout"

diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c

index d2f04bd..4a55cce 100644

--- a/rpc/rpc-transport/rdma/src/rdma.c

+++ b/rpc/rpc-transport/rdma/src/rdma.c

@@ -4498,6 +4498,12 @@ gf_rdma_options_init (rpc_transport_t *this)

         options->attr_retry_cnt = GF_RDMA_RETRY_CNT;

         options->attr_rnr_retry = GF_RDMA_RNR_RETRY;

+        temp = dict_get (this->options, "transport.listen-backlog");

+        if (temp)

+                options->backlog = data_to_uint32 (temp);

+        else

+                options->backlog = GLUSTERFS_SOCKET_LISTEN_BACKLOG;

+

         temp = dict_get (this->options,

                          "transport.rdma.work-request-send-count");

         if (temp)

@@ -4634,6 +4640,7 @@ gf_rdma_init (rpc_transport_t *this)

         priv->peer.recv_count = options->recv_count;

         priv->peer.send_size = options->send_size;

         priv->peer.recv_size = options->recv_size;

+        priv->backlog = options->backlog;

         priv->peer.trans = this;

         INIT_LIST_HEAD (&priv->peer.ioq);

@@ -4847,7 +4854,8 @@ gf_rdma_listen (rpc_transport_t *this)

                 goto err;

         }

-        ret = rdma_listen (peer->cm_id, 10);

+        ret = rdma_listen (peer->cm_id, priv->backlog);

+

         if (ret != 0) {

                 gf_msg (this->name, GF_LOG_WARNING, errno,

                         RDMA_MSG_LISTEN_FAILED,

@@ -4918,6 +4926,28 @@ init (rpc_transport_t *this)

         return 0;

 }

+int

+reconfigure (rpc_transport_t *this, dict_t *options)

+{

+        gf_rdma_private_t *priv          = NULL;

+        uint32_t          backlog        = 0;

+        int               ret            = -1;

+

+        GF_VALIDATE_OR_GOTO ("rdma", this, out);

+        GF_VALIDATE_OR_GOTO ("rdma", this->private, out);

+

+        priv = this->private;

+

+        if (dict_get_uint32 (options, "transport.listen-backlog",

+                             &backlog) == 0) {

+                priv->backlog = backlog;

+                gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "

+                        "transport.listen-backlog=%d", priv->backlog);

+        }

+        ret = 0;

+out:

+        return ret;

+}

 void

 fini (struct rpc_transport *this)

 {

diff --git a/rpc/rpc-transport/rdma/src/rdma.h b/rpc/rpc-transport/rdma/src/rdma.h

index 449861f..6bcf6bc 100644

--- a/rpc/rpc-transport/rdma/src/rdma.h

+++ b/rpc/rpc-transport/rdma/src/rdma.h

@@ -144,6 +144,7 @@ struct __gf_rdma_options {

 	uint8_t  attr_timeout;

 	uint8_t  attr_retry_cnt;

 	uint8_t  attr_rnr_retry;

+        uint32_t backlog;

 };

 typedef struct __gf_rdma_options gf_rdma_options_t;

@@ -383,6 +384,7 @@ struct __gf_rdma_private {

         pthread_mutex_t             recv_mutex;

         pthread_cond_t              recv_cond;

         gf_rdma_transport_entity_t  entity;

+        uint32_t                    backlog;

 };

 typedef struct __gf_rdma_private    gf_rdma_private_t;

diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c

index 16cba78..4560d2f 100644

--- a/rpc/rpc-transport/socket/src/socket.c

+++ b/rpc/rpc-transport/socket/src/socket.c

@@ -3524,10 +3524,7 @@ socket_listen (rpc_transport_t *this)

                         goto unlock;

                 }

-                if (priv->backlog)

-                        ret = listen (priv->sock, priv->backlog);

-                else

-                        ret = listen (priv->sock, 10);

+                ret = listen (priv->sock, priv->backlog);

                 if (ret == -1) {

                         gf_log (this->name, GF_LOG_ERROR,

@@ -3843,6 +3840,7 @@ reconfigure (rpc_transport_t *this, dict_t *options)

         int               ret           = 0;

         uint64_t          windowsize    = 0;

         uint32_t          timeout       = 0;

+        uint32_t          backlog       = 0;

         GF_VALIDATE_OR_GOTO ("socket", this, out);

         GF_VALIDATE_OR_GOTO ("socket", this->private, out);

@@ -3878,6 +3876,13 @@ reconfigure (rpc_transport_t *this, dict_t *options)

                         "transport.tcp-user-timeout=%d", timeout);

         }

+        if (dict_get_uint32 (options, "transport.listen-backlog",

+                             &backlog) == 0) {

+                priv->backlog = backlog;

+                gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "

+                        "transport.listen-backlog=%d", priv->backlog);

+        }

+

         optstr = NULL;

         if (dict_get_str (options, "tcp-window-size", &optstr) == 0) {

                 if (gf_string2uint64 (optstr, &windowsize) != 0) {

@@ -4144,14 +4149,17 @@ socket_init (rpc_transport_t *this)

                              &timeout) == 0) {

                 priv->timeout = timeout;

         }

+

         gf_log (this->name, GF_LOG_DEBUG, "Configued "

                 "transport.tcp-user-timeout=%d", priv->timeout);

         if (dict_get_uint32 (this->options,

-                             "transport.socket.listen-backlog",

-                             &backlog) == 0) {

-                priv->backlog = backlog;

+                             "transport.listen-backlog",

+                             &backlog) != 0) {

+

+                backlog = GLUSTERFS_SOCKET_LISTEN_BACKLOG;

         }

+        priv->backlog = backlog;

         optstr = NULL;

@@ -4561,9 +4569,6 @@ struct volume_options options[] = {

         { .key   = {"transport.socket.keepalive-time"},

           .type  = GF_OPTION_TYPE_INT

         },

-        { .key   = {"transport.socket.listen-backlog"},

-          .type  = GF_OPTION_TYPE_INT

-        },

         { .key   = {"transport.socket.read-fail-log"},

           .type  = GF_OPTION_TYPE_BOOL

         },

diff --git a/tests/basic/multiplex.t b/tests/basic/multiplex.t

index 819ff40..b69fe57 100644

--- a/tests/basic/multiplex.t

+++ b/tests/basic/multiplex.t

@@ -27,6 +27,9 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_up_bricks

 EXPECT 1 online_brick_count

 TEST $CLI volume stop $V0

+#Testing the volume set command introduced for protocol/server

+TEST $CLI volume set $V0 transport.listen-backlog 1024

+

 EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT 0 online_brick_count

 TEST $CLI volume start $V0

 EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_up_bricks

diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h

index 3d0f48c..58208ec 100644

--- a/xlators/mgmt/glusterd/src/glusterd-messages.h

+++ b/xlators/mgmt/glusterd/src/glusterd-messages.h

@@ -41,7 +41,7 @@

 #define GLUSTERD_COMP_BASE      GLFS_MSGID_GLUSTERD

-#define GLFS_NUM_MESSAGES       601

+#define GLFS_NUM_MESSAGES       602

 #define GLFS_MSGID_END          (GLUSTERD_COMP_BASE + GLFS_NUM_MESSAGES + 1)

 /* Messaged with message IDs */

@@ -4809,6 +4809,14 @@

  */

 #define GD_MSG_PIDFILE_UNLINKING                   (GLUSTERD_COMP_BASE + 601)

+/*!

+ * @messageid

+ * @diagnosis

+ * @recommendedaction

+ *

+ */

+#define GD_MSG_VOL_SET_VALIDATION_INFO             (GLUSTERD_COMP_BASE + 602)

+

 /*------------*/

 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"

diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c

index d5fa23b..de4cfb7 100644

--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c

+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c

@@ -707,6 +707,52 @@ out:

 }

 static int

+validate_server_options (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,

+                         char *value, char **op_errstr)

+{

+        char                 errstr[2048]  = "";

+        xlator_t            *this          = NULL;

+        int                  ret           = -1;

+        int                  origin_val    = 0;

+

+        this = THIS;

+        GF_ASSERT (this);

+

+        if (volinfo->status == GLUSTERD_STATUS_STARTED) {

+                gf_msg (this->name, GF_LOG_INFO, 0,

+                        GD_MSG_VOL_SET_VALIDATION_INFO, "Please note that "

+                        "volume %s is started. This option will only get "

+                        "effected after a brick restart.", volinfo->volname);

+        }

+

+        ret = gf_string2int (value, &origin_val);

+        if (ret) {

+                snprintf (errstr, sizeof (errstr), "%s is not a compatible "

+                          "value. %s expects an integer value.", value, key);

+                ret = -1;

+                goto out;

+        }

+

+        if (origin_val < 0) {

+                snprintf (errstr, sizeof (errstr), "%s is not a "

+                          "compatible value. %s expects a positive"

+                          "integer value.", value, key);

+                ret = -1;

+                goto out;

+        }

+

+        ret = 0;

+out:

+        if (ret) {

+                gf_msg (this->name, GF_LOG_ERROR, EINVAL,

+                        GD_MSG_INCOMPATIBLE_VALUE, "%s", errstr);

+                *op_errstr = gf_strdup (errstr);

+        }

+

+        return ret;

+}

+

+static int

 validate_stripe (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,

                  char *value, char **op_errstr)

 {

@@ -1919,6 +1965,16 @@ struct volopt_map_entry glusterd_volopt_map[] = {

           .op_version  = GD_OP_VERSION_3_7_4,

           .type        = NO_DOC,

         },

+        { .key         = "transport.listen-backlog",

+          .voltype     = "protocol/server",

+          .option      = "transport.listen-backlog",

+          .op_version  = GD_OP_VERSION_3_11_1,

+          .validate_fn = validate_server_options,

+          .description = "This option uses the value of backlog argument that "

+                         "defines the maximum length to which the queue of "

+                         "pending connections for socket fd may grow.",

+          .value       = "10",

+        },

         /* Performance xlators enable/disbable options */

         { .key         = "performance.write-behind",

diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c

index f718d36..e49b186 100644

--- a/xlators/mgmt/glusterd/src/glusterd.c

+++ b/xlators/mgmt/glusterd/src/glusterd.c

@@ -461,14 +461,12 @@ glusterd_rpcsvc_options_build (dict_t *options)

         int             ret = 0;

         uint32_t        backlog = 0;

-        ret = dict_get_uint32 (options, "transport.socket.listen-backlog",

-                               &backlog);

+        ret = dict_get_uint32 (options, "transport.listen-backlog", &backlog);

         if (ret) {

-                backlog = GLUSTERD_SOCKET_LISTEN_BACKLOG;

+                backlog = GLUSTERFS_SOCKET_LISTEN_BACKLOG;

                 ret = dict_set_uint32 (options,

-                                      "transport.socket.listen-backlog",

-                                      backlog);

+                                      "transport.listen-backlog", backlog);

                 if (ret)

                         goto out;

         }

diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h

index b5a0a3b..7b40196 100644

--- a/xlators/mgmt/glusterd/src/glusterd.h

+++ b/xlators/mgmt/glusterd/src/glusterd.h

@@ -39,7 +39,6 @@

 #include "events.h"

 #define GLUSTERD_TR_LOG_SIZE            50

-#define GLUSTERD_SOCKET_LISTEN_BACKLOG  128

 #define GLUSTERD_QUORUM_TYPE_KEY        "cluster.server-quorum-type"

 #define GLUSTERD_QUORUM_RATIO_KEY       "cluster.server-quorum-ratio"

 #define GLUSTERD_GLOBAL_OPT_VERSION     "global-option-version"

diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c

index 6acf256..5d4d860 100644

--- a/xlators/protocol/server/src/server.c

+++ b/xlators/protocol/server/src/server.c

@@ -1622,6 +1622,10 @@ struct volume_options options[] = {

                     "socket*([ \t]),*([ \t])rdma"},

           .type  = GF_OPTION_TYPE_STR

         },

+        { .key   = {"transport.listen-backlog"},

+          .type  = GF_OPTION_TYPE_INT,

+          .default_value = "10",

+        },

         { .key   = {"volume-filename.*"},

           .type  = GF_OPTION_TYPE_PATH,

         },

-- 

1.8.3.1