14f8ab
From a30a5fdef2e252eba9f44a3c671de8f3aa4f17d7 Mon Sep 17 00:00:00 2001
14f8ab
From: Vishal Pandey <vpandey@redhat.com>
14f8ab
Date: Tue, 19 Nov 2019 11:39:22 +0530
14f8ab
Subject: [PATCH 392/449] glusterd: Brick process fails to come up with
14f8ab
 brickmux on
14f8ab
14f8ab
Issue:
14f8ab
1- In a cluster of 3 Nodes N1, N2, N3. Create 3 volumes vol1,
14f8ab
vol2, vol3 with 3 bricks (one from each node)
14f8ab
2- Set cluster.brick-multiplex on
14f8ab
3- Start all 3 volumes
14f8ab
4- Check if all bricks on a node are running on same port
14f8ab
5- Kill N1
14f8ab
6- Set performance.readdir-ahead for volumes vol1, vol2, vol3
14f8ab
7- Bring N1 up and check volume status
14f8ab
8- All bricks processes not running on N1.
14f8ab
14f8ab
Root Cause -
14f8ab
Since, There is a diff in volfile versions in N1 as compared
14f8ab
to N2 and N3 therefore glusterd_import_friend_volume() is called.
14f8ab
glusterd_import_friend_volume() copies the new_volinfo and deletes
14f8ab
old_volinfo and then calls glusterd_start_bricks().
14f8ab
glusterd_start_bricks() looks for the volfiles and sends an rpc
14f8ab
request to glusterfs_handle_attach(). Now, since the volinfo
14f8ab
has been deleted by glusterd_delete_stale_volume()
14f8ab
from priv->volumes list before glusterd_start_bricks() and
14f8ab
glusterd_create_volfiles_and_notify_services() and
14f8ab
glusterd_list_add_order is called after glusterd_start_bricks(),
14f8ab
therefore the attach RPC req gets an empty volfile path
14f8ab
and that causes the brick to crash.
14f8ab
14f8ab
Fix- Call glusterd_list_add_order() and
14f8ab
glusterd_create_volfiles_and_notify_services before
14f8ab
glusterd_start_bricks() cal is made in glusterd_import_friend_volume
14f8ab
14f8ab
> upstream patch link: https://review.gluster.org/#/c/glusterfs/+/23724/
14f8ab
> Change-Id: Idfe0e8710f7eb77ca3ddfa1cabeb45b2987f41aa
14f8ab
> Fixes: bz#1773856
14f8ab
> Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
14f8ab
14f8ab
BUG: 1683602
14f8ab
Change-Id: Idfe0e8710f7eb77ca3ddfa1cabeb45b2987f41aa
14f8ab
Signed-off-by: Sanju Rakonde <srakonde@redhat.com>
14f8ab
Reviewed-on: https://code.engineering.redhat.com/gerrit/202255
14f8ab
Tested-by: RHGS Build Bot <nigelb@redhat.com>
14f8ab
Reviewed-by: Mohit Agrawal <moagrawa@redhat.com>
14f8ab
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
14f8ab
---
14f8ab
 .../glusterd/brick-mux-validation-in-cluster.t     | 61 +++++++++++++++++++++-
14f8ab
 xlators/mgmt/glusterd/src/glusterd-utils.c         | 28 +++++-----
14f8ab
 2 files changed, 75 insertions(+), 14 deletions(-)
14f8ab
14f8ab
diff --git a/tests/bugs/glusterd/brick-mux-validation-in-cluster.t b/tests/bugs/glusterd/brick-mux-validation-in-cluster.t
14f8ab
index 4e57038..f088dbb 100644
14f8ab
--- a/tests/bugs/glusterd/brick-mux-validation-in-cluster.t
14f8ab
+++ b/tests/bugs/glusterd/brick-mux-validation-in-cluster.t
14f8ab
@@ -7,6 +7,20 @@ function count_brick_processes {
14f8ab
         pgrep glusterfsd | wc -l
14f8ab
 }
14f8ab
 
14f8ab
+function count_brick_pids {
14f8ab
+        $CLI_1 --xml volume status all | sed -n '/.*<pid>\([^<]*\).*/s//\1/p' \
14f8ab
+                                     | grep -v "N/A" | sort | uniq | wc -l
14f8ab
+}
14f8ab
+
14f8ab
+function count_N/A_brick_pids {
14f8ab
+        $CLI_1 --xml volume status all | sed -n '/.*<pid>\([^<]*\).*/s//\1/p' \
14f8ab
+                                     | grep -- '\-1' | sort | uniq | wc -l
14f8ab
+}
14f8ab
+
14f8ab
+function check_peers {
14f8ab
+        $CLI_2 peer status | grep 'Peer in Cluster (Connected)' | wc -l
14f8ab
+}
14f8ab
+
14f8ab
 cleanup;
14f8ab
 
14f8ab
 TEST launch_cluster 3
14f8ab
@@ -48,4 +62,49 @@ TEST $CLI_1 volume stop $V1
14f8ab
 
14f8ab
 EXPECT 3 count_brick_processes
14f8ab
 
14f8ab
-cleanup
14f8ab
+TEST $CLI_1 volume stop $META_VOL
14f8ab
+
14f8ab
+TEST $CLI_1 volume delete $META_VOL
14f8ab
+TEST $CLI_1 volume delete $V0
14f8ab
+TEST $CLI_1 volume delete $V1
14f8ab
+
14f8ab
+#bug-1773856 - Brick process fails to come up with brickmux on
14f8ab
+
14f8ab
+TEST $CLI_1 volume create $V0 $H1:$B1/${V0}1 $H2:$B2/${V0}1 $H3:$B3/${V0}1 force
14f8ab
+TEST $CLI_1 volume start $V0
14f8ab
+
14f8ab
+
14f8ab
+EXPECT 3 count_brick_processes
14f8ab
+
14f8ab
+#create and start a new volume
14f8ab
+TEST $CLI_1 volume create $V1 $H1:$B1/${V1}2 $H2:$B2/${V1}2 $H3:$B3/${V1}2 force
14f8ab
+TEST $CLI_1 volume start $V1
14f8ab
+
14f8ab
+EXPECT 3 count_brick_processes
14f8ab
+
14f8ab
+V2=patchy2
14f8ab
+TEST $CLI_1 volume create $V2 $H1:$B1/${V2}3 $H2:$B2/${V2}3 $H3:$B3/${V2}3 force
14f8ab
+TEST $CLI_1 volume start $V2
14f8ab
+
14f8ab
+EXPECT 3 count_brick_processes
14f8ab
+
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_brick_pids
14f8ab
+
14f8ab
+TEST kill_node 1
14f8ab
+
14f8ab
+sleep 10
14f8ab
+
14f8ab
+EXPECT_WITHIN $PROBE_TIMEOUT 1 check_peers;
14f8ab
+
14f8ab
+$CLI_2 volume set $V0 performance.readdir-ahead on
14f8ab
+$CLI_2 volume set $V1 performance.readdir-ahead on
14f8ab
+
14f8ab
+TEST $glusterd_1;
14f8ab
+
14f8ab
+sleep 10
14f8ab
+
14f8ab
+EXPECT 4 count_brick_processes
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 4 count_brick_pids
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0  count_N/A_brick_pids
14f8ab
+
14f8ab
+cleanup;
14f8ab
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
14f8ab
index 6654741..1b78812 100644
14f8ab
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
14f8ab
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
14f8ab
@@ -4988,16 +4988,6 @@ glusterd_import_friend_volume(dict_t *peer_data, int count)
14f8ab
         glusterd_volinfo_unref(old_volinfo);
14f8ab
     }
14f8ab
 
14f8ab
-    if (glusterd_is_volume_started(new_volinfo)) {
14f8ab
-        (void)glusterd_start_bricks(new_volinfo);
14f8ab
-        if (glusterd_is_snapd_enabled(new_volinfo)) {
14f8ab
-            svc = &(new_volinfo->snapd.svc);
14f8ab
-            if (svc->manager(svc, new_volinfo, PROC_START_NO_WAIT)) {
14f8ab
-                gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
14f8ab
-            }
14f8ab
-        }
14f8ab
-    }
14f8ab
-
14f8ab
     ret = glusterd_store_volinfo(new_volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
14f8ab
     if (ret) {
14f8ab
         gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
14f8ab
@@ -5007,19 +4997,31 @@ glusterd_import_friend_volume(dict_t *peer_data, int count)
14f8ab
         goto out;
14f8ab
     }
14f8ab
 
14f8ab
-    ret = glusterd_create_volfiles_and_notify_services(new_volinfo);
14f8ab
+    ret = glusterd_create_volfiles(new_volinfo);
14f8ab
     if (ret)
14f8ab
         goto out;
14f8ab
 
14f8ab
+    glusterd_list_add_order(&new_volinfo->vol_list, &priv->volumes,
14f8ab
+                            glusterd_compare_volume_name);
14f8ab
+
14f8ab
+    if (glusterd_is_volume_started(new_volinfo)) {
14f8ab
+        (void)glusterd_start_bricks(new_volinfo);
14f8ab
+        if (glusterd_is_snapd_enabled(new_volinfo)) {
14f8ab
+            svc = &(new_volinfo->snapd.svc);
14f8ab
+            if (svc->manager(svc, new_volinfo, PROC_START_NO_WAIT)) {
14f8ab
+                gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
14f8ab
+            }
14f8ab
+        }
14f8ab
+    }
14f8ab
+
14f8ab
     ret = glusterd_import_quota_conf(peer_data, count, new_volinfo, "volume");
14f8ab
     if (ret) {
14f8ab
         gf_event(EVENT_IMPORT_QUOTA_CONF_FAILED, "volume=%s",
14f8ab
                  new_volinfo->volname);
14f8ab
         goto out;
14f8ab
     }
14f8ab
-    glusterd_list_add_order(&new_volinfo->vol_list, &priv->volumes,
14f8ab
-                            glusterd_compare_volume_name);
14f8ab
 
14f8ab
+    ret = glusterd_fetchspec_notify(this);
14f8ab
 out:
14f8ab
     gf_msg_debug("glusterd", 0, "Returning with ret: %d", ret);
14f8ab
     return ret;
14f8ab
-- 
14f8ab
1.8.3.1
14f8ab