From f37a409a8c0fa683ad95a61bf71e949f215e2f81 Mon Sep 17 00:00:00 2001

From: Gaurav Yadav <gyadav@redhat.com>

Date: Thu, 5 Oct 2017 23:44:46 +0530

Subject: [PATCH 48/74] glusterd : introduce timer in mgmt_v3_lock

Problem:

In a multinode environment, if two of the op-sm transactions

are initiated on one of the receiver nodes at the same time,

there might be a possibility that glusterd  may end up in

stale lock.

Solution:

During mgmt_v3_lock a registration is made to  gf_timer_call_after

which release the lock after certain period of time

>mainline patch : https://review.gluster.org/#/c/18437

Change-Id: I16cc2e5186a2e8a5e35eca2468b031811e093843

BUG: 1442983

Signed-off-by: Gaurav Yadav <gyadav@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/123069

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

---

 extras/glusterd.vol.in                     |   1 +

 libglusterfs/src/common-utils.h            |   2 +-

 libglusterfs/src/mem-types.h               |   1 +

 xlators/mgmt/glusterd/src/glusterd-locks.c | 219 +++++++++++++++++++++++++++--

 xlators/mgmt/glusterd/src/glusterd-locks.h |  13 ++

 xlators/mgmt/glusterd/src/glusterd.c       |  28 +++-

 xlators/mgmt/glusterd/src/glusterd.h       |   2 +

 7 files changed, 246 insertions(+), 20 deletions(-)

diff --git a/extras/glusterd.vol.in b/extras/glusterd.vol.in

index 0152996..fe413a9 100644

--- a/extras/glusterd.vol.in

+++ b/extras/glusterd.vol.in

@@ -7,6 +7,7 @@ volume management

     option transport.socket.read-fail-log off

     option ping-timeout 0

     option event-threads 1

+#   option lock-timer 180

 #   option transport.address-family inet6

 #   option base-port 49152

 #   option max-port  65535

diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h

index e1c5f66..0131070 100644

--- a/libglusterfs/src/common-utils.h

+++ b/libglusterfs/src/common-utils.h

@@ -102,7 +102,7 @@ void trap (void);

 #define GF_CLNT_INSECURE_PORT_CEILING (GF_IANA_PRIV_PORTS_START - 1)

 #define GF_PORT_MAX 65535

 #define GF_PORT_ARRAY_SIZE ((GF_PORT_MAX + 7) / 8)

-

+#define GF_LOCK_TIMER 180

 #define GF_MINUTE_IN_SECONDS 60

 #define GF_HOUR_IN_SECONDS (60*60)

 #define GF_DAY_IN_SECONDS (24*60*60)

diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h

index d244fb5..85cb5d2 100644

--- a/libglusterfs/src/mem-types.h

+++ b/libglusterfs/src/mem-types.h

@@ -177,6 +177,7 @@ enum gf_common_mem_types_ {

         gf_common_mt_pthread_t,

         gf_common_ping_local_t,

         gf_common_volfile_t,

+        gf_common_mt_mgmt_v3_lock_timer_t,

         gf_common_mt_end

 };

 #endif

diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.c b/xlators/mgmt/glusterd/src/glusterd-locks.c

index 146092d..bd73b37 100644

--- a/xlators/mgmt/glusterd/src/glusterd-locks.c

+++ b/xlators/mgmt/glusterd/src/glusterd-locks.c

@@ -94,6 +94,50 @@ glusterd_mgmt_v3_lock_fini ()

                 dict_unref (priv->mgmt_v3_lock);

 }

+/* Initialize the global mgmt_v3_timer lock list(dict) when

+ * glusterd is spawned */

+int32_t

+glusterd_mgmt_v3_lock_timer_init ()

+{

+        int32_t             ret = -1;

+        xlator_t           *this   = NULL;

+        glusterd_conf_t    *priv   = NULL;

+

+        this = THIS;

+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);

+

+        priv = this->private;

+        GF_VALIDATE_OR_GOTO (this->name, priv, out);

+

+        priv->mgmt_v3_lock_timer = dict_new ();

+        if (!priv->mgmt_v3_lock_timer)

+                goto out;

+

+        ret = 0;

+out:

+        return ret;

+}

+

+/* Destroy the global mgmt_v3_timer lock list(dict) when

+ * glusterd cleanup is performed */

+void

+glusterd_mgmt_v3_lock_timer_fini ()

+{

+        xlator_t           *this   = NULL;

+        glusterd_conf_t    *priv   = NULL;

+

+        this = THIS;

+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);

+

+        priv = this->private;

+        GF_VALIDATE_OR_GOTO (this->name, priv, out);

+

+        if (priv->mgmt_v3_lock_timer)

+                dict_unref (priv->mgmt_v3_lock_timer);

+out:

+        return;

+}

+

 int32_t

 glusterd_get_mgmt_v3_lock_owner (char *key, uuid_t *uuid)

 {

@@ -513,17 +557,23 @@ int32_t

 glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, uint32_t *op_errno,

                        char *type)

 {

-        char                            key[PATH_MAX]   = "";

-        int32_t                         ret             = -1;

-        glusterd_mgmt_v3_lock_obj      *lock_obj        = NULL;

-        glusterd_conf_t                *priv            = NULL;

-        gf_boolean_t                    is_valid        = _gf_true;

-        uuid_t                          owner           = {0};

-        xlator_t                       *this            = NULL;

-        char                           *bt              = NULL;

+        char                            key[PATH_MAX]       = "";

+        int32_t                         ret                 = -1;

+        glusterd_mgmt_v3_lock_obj      *lock_obj            = NULL;

+        glusterd_mgmt_v3_lock_timer    *mgmt_lock_timer     = NULL;

+        glusterd_conf_t                *priv                = NULL;

+        gf_boolean_t                    is_valid            = _gf_true;

+        uuid_t                          owner               = {0};

+        xlator_t                       *this                = NULL;

+        char                           *bt                  = NULL;

+        struct timespec                 delay               = {0};

+        char                           *key_dup             = NULL;

+        glusterfs_ctx_t                *mgmt_lock_timer_ctx = NULL;

+        xlator_t                       *mgmt_lock_timer_xl  = NULL;

         this = THIS;

         GF_ASSERT (this);

+

         priv = this->private;

         GF_ASSERT (priv);

@@ -594,6 +644,42 @@ glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, uint32_t *op_errno,

                 goto out;

         }

+        mgmt_lock_timer = GF_CALLOC (1, sizeof(glusterd_mgmt_v3_lock_timer),

+                                     gf_common_mt_mgmt_v3_lock_timer_t);

+

+        if (!mgmt_lock_timer) {

+                ret = -1;

+                goto out;

+        }

+

+        mgmt_lock_timer->xl = THIS;

+        key_dup = gf_strdup (key);

+        delay.tv_sec = priv->mgmt_v3_lock_timeout;

+        delay.tv_nsec = 0;

+

+        ret = -1;

+        mgmt_lock_timer_xl = mgmt_lock_timer->xl;

+        GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_xl, out);

+

+        mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx;

+        GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_ctx, out);

+

+        mgmt_lock_timer->timer = gf_timer_call_after

+                                     (mgmt_lock_timer_ctx, delay,

+                                      gd_mgmt_v3_unlock_timer_cbk,

+                                      key_dup);

+

+        ret = dict_set_bin (priv->mgmt_v3_lock_timer, key, mgmt_lock_timer,

+                            sizeof (glusterd_mgmt_v3_lock_timer));

+        if (ret) {

+                gf_msg (this->name, GF_LOG_ERROR, 0,

+                        GD_MSG_DICT_SET_FAILED,

+                        "Unable to set timer in mgmt_v3 lock");

+                GF_FREE (mgmt_lock_timer);

+                goto out;

+        }

+

+

         /* Saving the backtrace into the pre-allocated buffer, ctx->btbuf*/

         if ((bt = gf_backtrace_save (NULL))) {

                 snprintf (key, sizeof (key), "debug.last-success-bt-%s-%s",

@@ -617,18 +703,98 @@ out:

         return ret;

 }

+/*

+ * This call back will ensure to unlock the lock_obj, in case we hit a situation

+ * where unlocking failed and stale lock exist*/

+void

+gd_mgmt_v3_unlock_timer_cbk (void *data)

+{

+        xlator_t                        *this               = NULL;

+        glusterd_conf_t                 *conf               = NULL;

+        glusterd_mgmt_v3_lock_timer     *mgmt_lock_timer    = NULL;

+        char                            *key                = NULL;

+        char                            *type               = NULL;

+        char                            bt_key[PATH_MAX]    = "";

+        char                            name[PATH_MAX]      = "";

+        int32_t                         ret                 = -1;

+        glusterfs_ctx_t                *mgmt_lock_timer_ctx = NULL;

+        xlator_t                       *mgmt_lock_timer_xl  = NULL;

+

+        this = THIS;

+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);

+

+        conf = this->private;

+        GF_VALIDATE_OR_GOTO (this->name, conf, out);

+

+        gf_log (THIS->name, GF_LOG_INFO, "In gd_mgmt_v3_unlock_timer_cbk");

+        GF_ASSERT (NULL != data);

+        key = (char *)data;

+

+        dict_del (conf->mgmt_v3_lock, key);

+

+        type = strrchr (key, '_');

+        strncpy (name, key, strlen (key) - strlen (type) - 1);

+

+        ret = snprintf (bt_key, PATH_MAX, "debug.last-success-bt-%s-%s",

+                        name, type + 1);

+        if (ret != strlen ("debug.last-success-bt-") + strlen (name) +

+                   strlen (type)) {

+                gf_msg (this->name, GF_LOG_ERROR, 0,

+                        GD_MSG_CREATE_KEY_FAIL, "Unable to create backtrace "

+                        "key");

+                goto out;

+        }

+

+        dict_del (conf->mgmt_v3_lock, bt_key);

+

+        ret = dict_get_bin (conf->mgmt_v3_lock_timer, key,

+                            (void **)&mgmt_lock_timer);

+        if (ret) {

+                gf_msg (this->name, GF_LOG_ERROR, 0,

+                        GD_MSG_DICT_SET_FAILED,

+                        "Unable to get lock owner in mgmt_v3 lock");

+                goto out;

+        }

+

+out:

+        if (mgmt_lock_timer->timer) {

+                mgmt_lock_timer_xl = mgmt_lock_timer->xl;

+                GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_xl,

+                                     ret_function);

+

+                mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx;

+                GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_ctx,

+                                     ret_function);

+

+                gf_timer_call_cancel (mgmt_lock_timer_ctx,

+                                      mgmt_lock_timer->timer);

+                GF_FREE(key);

+                dict_del (conf->mgmt_v3_lock_timer, bt_key);

+                mgmt_lock_timer->timer = NULL;

+        }

+

+ret_function:

+

+        return;

+}

+

 int32_t

 glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type)

 {

-        char                    key[PATH_MAX]   = "";

-        int32_t                 ret             = -1;

-        gf_boolean_t            is_valid        = _gf_true;

-        glusterd_conf_t        *priv            = NULL;

-        uuid_t                  owner           = {0};

-        xlator_t               *this            = NULL;

+        char                            key[PATH_MAX]       = "";

+        char                            key_dup[PATH_MAX]   = "";

+        int32_t                         ret                 = -1;

+        gf_boolean_t                    is_valid            = _gf_true;

+        glusterd_conf_t                 *priv               = NULL;

+        glusterd_mgmt_v3_lock_timer     *mgmt_lock_timer    = NULL;

+        uuid_t                          owner               = {0};

+        xlator_t                        *this               = NULL;

+        glusterfs_ctx_t                *mgmt_lock_timer_ctx = NULL;

+        xlator_t                       *mgmt_lock_timer_xl  = NULL;

         this = THIS;

         GF_ASSERT (this);

+

         priv = this->private;

         GF_ASSERT (priv);

@@ -657,6 +823,7 @@ glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type)

                 ret = -1;

                 goto out;

         }

+        strncpy (key_dup, key, strlen(key));

         gf_msg_debug (this->name, 0,

                 "Trying to release lock of %s %s for %s as %s",

@@ -690,6 +857,15 @@ glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type)

         /* Removing the mgmt_v3 lock from the global list */

         dict_del (priv->mgmt_v3_lock, key);

+        ret = dict_get_bin (priv->mgmt_v3_lock_timer, key,

+                            (void **)&mgmt_lock_timer);

+        if (ret) {

+                gf_msg (this->name, GF_LOG_ERROR, 0,

+                        GD_MSG_DICT_SET_FAILED,

+                        "Unable to get mgmt lock key in mgmt_v3 lock");

+                goto out;

+        }

+

         /* Remove the backtrace key as well */

         ret = snprintf (key, sizeof(key), "debug.last-success-bt-%s-%s", name,

                         type);

@@ -708,7 +884,22 @@ glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type)

                 type, name);

         ret = 0;

+        /* Release owner refernce which was held during lock */

+        if (mgmt_lock_timer->timer) {

+                ret = -1;

+                mgmt_lock_timer_xl = mgmt_lock_timer->xl;

+                GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_xl, out);

+

+                mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx;

+                GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_ctx, out);

+                ret = 0;

+                gf_timer_call_cancel (mgmt_lock_timer_ctx,

+                                      mgmt_lock_timer->timer);

+                dict_del (priv->mgmt_v3_lock_timer, key_dup);

+                mgmt_lock_timer->timer = NULL;

+        }

 out:

+

         gf_msg_trace (this->name, 0, "Returning %d", ret);

         return ret;

 }

diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.h b/xlators/mgmt/glusterd/src/glusterd-locks.h

index 437053d..226d5c6 100644

--- a/xlators/mgmt/glusterd/src/glusterd-locks.h

+++ b/xlators/mgmt/glusterd/src/glusterd-locks.h

@@ -14,6 +14,11 @@ typedef struct glusterd_mgmt_v3_lock_object_ {

         uuid_t              lock_owner;

 } glusterd_mgmt_v3_lock_obj;

+typedef struct glusterd_mgmt_v3_lock_timer_ {

+        gf_timer_t *timer;

+        xlator_t      *xl;

+} glusterd_mgmt_v3_lock_timer;

+

 typedef struct glusterd_mgmt_v3_lock_valid_entities {

         char          *type;          /* Entity type like vol, snap */

         gf_boolean_t   default_value; /* The default value that  *

@@ -29,6 +34,12 @@ void

 glusterd_mgmt_v3_lock_fini ();

 int32_t

+glusterd_mgmt_v3_lock_timer_init ();

+

+void

+glusterd_mgmt_v3_lock_timer_fini ();

+

+int32_t

 glusterd_get_mgmt_v3_lock_owner (char *volname, uuid_t *uuid);

 int32_t

@@ -44,4 +55,6 @@ glusterd_multiple_mgmt_v3_lock (dict_t *dict, uuid_t uuid, uint32_t *op_errno);

 int32_t

 glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid);

+void

+gd_mgmt_v3_unlock_timer_cbk(void *data);

 #endif

diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c

index 6ce4156..ed01b93 100644

--- a/xlators/mgmt/glusterd/src/glusterd.c

+++ b/xlators/mgmt/glusterd/src/glusterd.c

@@ -1858,14 +1858,22 @@ init (xlator_t *this)

                 gf_msg (this->name, GF_LOG_INFO, 0,

                         GD_MSG_DICT_SET_FAILED,

                         "base-port override: %d", conf->base_port);

-         }

-         conf->max_port = GF_PORT_MAX;

-         if (dict_get_uint32 (this->options, "max-port",

-                              &conf->max_port) == 0) {

+        }

+        conf->max_port = GF_PORT_MAX;

+        if (dict_get_uint32 (this->options, "max-port",

+                             &conf->max_port) == 0) {

                 gf_msg (this->name, GF_LOG_INFO, 0,

                         GD_MSG_DICT_SET_FAILED,

                         "max-port override: %d", conf->max_port);

-         }

+        }

+

+        conf->mgmt_v3_lock_timeout = GF_LOCK_TIMER;

+        if (dict_get_uint32 (this->options, "lock-timer",

+                             &conf->mgmt_v3_lock_timeout) == 0) {

+                gf_msg (this->name, GF_LOG_INFO, 0,

+                        GD_MSG_DICT_SET_FAILED,

+                        "lock-timer override: %d", conf->mgmt_v3_lock_timeout);

+        }

         /* Set option to run bricks on valgrind if enabled in glusterd.vol */

         this->ctx->cmd_args.valgrind = valgrind;

@@ -1891,6 +1899,7 @@ init (xlator_t *this)

         this->private = conf;

         glusterd_mgmt_v3_lock_init ();

+        glusterd_mgmt_v3_lock_timer_init();

         glusterd_txn_opinfo_dict_init ();

         glusterd_svcs_build ();

@@ -2048,6 +2057,7 @@ fini (xlator_t *this)

                 gf_store_handle_destroy (conf->handle);

         glusterd_sm_tr_log_delete (&conf->op_sm_log);

         glusterd_mgmt_v3_lock_fini ();

+        glusterd_mgmt_v3_lock_timer_fini ();

         glusterd_txn_opinfo_dict_fini ();

         GF_FREE (conf);

@@ -2171,6 +2181,14 @@ struct volume_options options[] = {

           .max = GF_PORT_MAX,

           .description = "Sets the max port for portmap query"

         },

+        { .key = {"mgmt-v3-lock-timeout"},

+          .type = GF_OPTION_TYPE_INT,

+          .max = 600,

+          .description = "Sets the mgmt-v3-lock-timeout for transactions."

+                         "Specifes the default timeout value after which "

+                         "lock acquired while performing transaction will "

+                         "be released."

+        },

         { .key = {"snap-brick-path"},

           .type = GF_OPTION_TYPE_STR,

           .description = "directory where the bricks for the snapshots will be created"

diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h

index 291f2f7..59b1775 100644

--- a/xlators/mgmt/glusterd/src/glusterd.h

+++ b/xlators/mgmt/glusterd/src/glusterd.h

@@ -174,6 +174,7 @@ typedef struct {

                                                  * cluster with no

                                                  * transaction ids */

+        dict_t                    *mgmt_v3_lock_timer;

         struct cds_list_head       mount_specs;

         pthread_t                  brick_thread;

         void                      *hooks_priv;

@@ -195,6 +196,7 @@ typedef struct {

         uint32_t                   generation;

         int32_t                    workers;

         uint32_t                   blockers;

+        uint32_t                   mgmt_v3_lock_timeout;

 } glusterd_conf_t;

-- 

1.8.3.1