From cb565207cb7a3e7a9182bbed57a231f12bcaae0e Mon Sep 17 00:00:00 2001
From: Mohit Agrawal <moagrawa@redhat.com>
Date: Thu, 20 Sep 2018 18:11:36 +0530
Subject: [PATCH 391/399] glusterd: Use GF_ATOMIC to update 'blockers' counter
 at glusterd_conf

Problem: Currently in glusterd code uses sync_lock/sync_unlock to
         update blockers counter which could add delays to the overall
         transaction phase escpecially when there's a batch of volume
         stop operations processed by glusterd in brick multiplexing mode.

Solution: Use GF_ATOMIC to update blocker counter to ensure unnecessary
          context switching can be avoided.

> Change-Id: Ie13177dfee2af66687ae7cf5c67405c152853990
> Fixes: bz#1631128
> (Cherry picked from commit 4f6ae853ffa9d06446407f389aaef61ac0b3b424)
> (Reviewed on upstream link https://review.gluster.org/#/c/glusterfs/+/21221/)

Change-Id: I3023bce5ba50bc04e078c56ba6fa62a5b791d205
BUG: 1631329
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/150641
Tested-by: RHGS Build Bot <nigelb@redhat.com>
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
---
 xlators/mgmt/glusterd/src/glusterd-op-sm.c |  4 +++-
 xlators/mgmt/glusterd/src/glusterd-utils.c | 15 +++++----------
 xlators/mgmt/glusterd/src/glusterd.c       |  2 +-
 xlators/mgmt/glusterd/src/glusterd.h       |  2 +-
 4 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 6dfd819..df5b5c2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -6259,9 +6259,11 @@ glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
 static void
 glusterd_wait_for_blockers (glusterd_conf_t *priv)
 {
-        while (priv->blockers) {
+        uint64_t blockers = GF_ATOMIC_GET(priv->blockers);
+        while (blockers) {
                 synclock_unlock (&priv->big_lock);
                 sleep (1);
+                blockers = GF_ATOMIC_GET(priv->blockers);
                 synclock_lock (&priv->big_lock);
         }
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 2a176be..04fae63 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -5452,10 +5452,7 @@ my_callback (struct rpc_req *req, struct iovec *iov, int count, void *v_frame)
         call_frame_t    *frame  = v_frame;
         glusterd_conf_t *conf   = frame->this->private;
 
-        synclock_lock (&conf->big_lock);
-        --(conf->blockers);
-        synclock_unlock (&conf->big_lock);
-
+        GF_ATOMIC_DEC(conf->blockers);
         STACK_DESTROY (frame->root);
         return 0;
 }
@@ -5546,9 +5543,7 @@ attach_brick_callback (struct rpc_req *req, struct iovec *iov, int count,
                 }
         }
 out:
-        synclock_lock (&conf->big_lock);
-        --(conf->blockers);
-        synclock_unlock (&conf->big_lock);
+        GF_ATOMIC_DEC(conf->blockers);
         STACK_DESTROY (frame->root);
         return 0;
 }
@@ -5633,7 +5628,7 @@ send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path,
                 cbkfn = attach_brick_callback;
         }
         /* Send the msg */
-        ++(conf->blockers);
+        GF_ATOMIC_INC(conf->blockers);
         ret = rpc_clnt_submit (rpc, &gd_brick_prog, op,
                                cbkfn, &iov, 1, NULL, 0, iobref,
                                frame, NULL, 0, NULL, 0, NULL);
@@ -6380,7 +6375,7 @@ glusterd_restart_bricks (void *opaque)
         }
         conf->restart_bricks = _gf_true;
 
-        ++(conf->blockers);
+        GF_ATOMIC_INC(conf->blockers);
         ret = glusterd_get_quorum_cluster_counts (this, &active_count,
                                                   &quorum_count);
         if (ret)
@@ -6497,7 +6492,7 @@ glusterd_restart_bricks (void *opaque)
         ret = 0;
 
 out:
-        --(conf->blockers);
+        GF_ATOMIC_DEC(conf->blockers);
         conf->restart_done = _gf_true;
         conf->restart_bricks = _gf_false;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 78a37eb..076019f 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1979,7 +1979,7 @@ init (xlator_t *this)
                 }
         }
 
-        conf->blockers = 0;
+        GF_ATOMIC_INIT(conf->blockers, 0);
         /* If the peer count is less than 2 then this would be the best time to
          * spawn process/bricks that may need (re)starting since last time
          * (this) glusterd was up. */
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index d4f4f7e..8c70d48 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -198,7 +198,7 @@ typedef struct {
         int                        ping_timeout;
         uint32_t                   generation;
         int32_t                    workers;
-        uint32_t                   blockers;
+        gf_atomic_t                blockers;
         uint32_t                   mgmt_v3_lock_timeout;
         gf_boolean_t               restart_bricks;
 } glusterd_conf_t;
-- 
1.8.3.1