From 53ecd916d5ef56e164228ba123b078d4b30bfa81 Mon Sep 17 00:00:00 2001
From: Mohit Agrawal <moagrawal@redhat.com>
Date: Thu, 12 Jul 2018 13:29:48 +0530
Subject: [PATCH 333/333] glusterd: Add multiple checks before attach/start a
brick
Problem: In brick mux scenario sometime glusterd is not able
to start/attach a brick and gluster v status shows
brick is already running
Solution:
1) To make sure brick is running check brick_path in
/proc/<pid>/fd , if a brick is consumed by the brick
process it means brick stack is come up otherwise not
2) Before start/attach a brick check if a brick is mounted
or not
3) At the time of printing volume status check brick is
consumed by any brick process
Test: To test the same followed procedure
1) Setup brick mux environment on a vm
2) Put a breaking point in gdb in function posix_health_check_thread_proc
at the time of notify GF_EVENT_CHILD_DOWN event
3) unmount anyone brick path forcefully
4) check gluster v status it will show N/A for the brick
5) Try to start volume with force option, glusterd throw
message "No device available for mount brick"
6) Mount the brick_root path
7) Try to start volume with force option
8) down brick is started successfully
> Change-Id: I91898dad21d082ebddd12aa0d1f7f0ed012bdf69
> fixes: bz#1595320
> (cherry picked from commit 9400b6f2c8aa219a493961e0ab9770b7f12e80d2)
> (Reviewed on upstream link https://review.gluster.org/#/c/20202/)
Change-Id: I62459910272754e4e062b2725fea2a1e68d743f1
BUG: 1589279
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/145269
Tested-by: RHGS Build Bot <nigelb@redhat.com>
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
---
glusterfsd/src/glusterfsd-mgmt.c | 3 +
tests/basic/bug-1595320.t | 92 +++++++++
tests/basic/posix/shared-statfs.t | 2 +
tests/bitrot/bug-1373520.t | 1 +
tests/bugs/distribute/bug-1368012.t | 2 +
tests/bugs/distribute/bug-853258.t | 1 +
tests/bugs/quota/bug-1293601.t | 3 +-
xlators/mgmt/glusterd/src/glusterd-snapshot.c | 2 +-
xlators/mgmt/glusterd/src/glusterd-utils.c | 261 ++++++++++++++++++++----
xlators/mgmt/glusterd/src/glusterd-utils.h | 6 +-
xlators/mgmt/glusterd/src/glusterd-volume-ops.c | 7 +-
11 files changed, 329 insertions(+), 51 deletions(-)
create mode 100644 tests/basic/bug-1595320.t
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 30a717f..cbd436a 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1010,6 +1010,9 @@ glusterfs_handle_attach (rpcsvc_request_t *req)
"got attach for %s but no active graph",
xlator_req.name);
}
+ if (ret) {
+ ret = -1;
+ }
glusterfs_translator_info_response_send (req, ret, NULL, NULL);
diff --git a/tests/basic/bug-1595320.t b/tests/basic/bug-1595320.t
new file mode 100644
index 0000000..9d856ee
--- /dev/null
+++ b/tests/basic/bug-1595320.t
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../snapshot.rc
+
+cleanup
+
+function count_up_bricks {
+ $CLI --xml volume status $V0 | grep '<status>1' | wc -l
+}
+
+function count_brick_processes {
+ pgrep glusterfsd | wc -l
+}
+
+# Setup 3 LVMS
+LVM_PREFIX="test"
+TEST init_n_bricks 3
+TEST setup_lvm 3
+
+# Start glusterd
+TEST glusterd
+TEST pidof glusterd
+
+# Create volume and enable brick multiplexing
+TEST $CLI volume create $V0 $H0:$L1 $H0:$L2 $H0:$L3
+gluster v set all cluster.brick-multiplex on
+
+# Start the volume
+TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_up_bricks
+EXPECT 1 count_brick_processes
+
+# Kill volume ungracefully
+brick_pid=`pgrep glusterfsd`
+
+# Make sure every brick root should be consumed by a brick process
+n=`ls -lrth /proc/$brick_pid/fd | grep -iw $L1 | grep -v ".glusterfs" | wc -l`
+TEST [ $n -eq 1 ]
+n=`ls -lrth /proc/$brick_pid/fd | grep -iw $L2 | grep -v ".glusterfs" | wc -l`
+TEST [ $n -eq 1 ]
+n=`ls -lrth /proc/$brick_pid/fd | grep -iw $L3 | grep -v ".glusterfs" | wc -l`
+TEST [ $n -eq 1 ]
+
+b1_pid_file=$(ls $GLUSTERD_PIDFILEDIR/vols/$V0/*d-backends-1*.pid)
+b2_pid_file=$(ls $GLUSTERD_PIDFILEDIR/vols/$V0/*d-backends-2*.pid)
+b3_pid_file=$(ls $GLUSTERD_PIDFILEDIR/vols/$V0/*d-backends-3*.pid)
+
+kill -9 $brick_pid
+EXPECT 0 count_brick_processes
+
+# Unmount 3rd brick root from node
+brick_root=$L3
+TEST umount -l $brick_root 2>/dev/null
+
+# Start the volume only 2 brick should be start
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_up_bricks
+EXPECT 1 count_brick_processes
+
+brick_pid=`pgrep glusterfsd`
+
+# Make sure only two brick root should be consumed by a brick process
+n=`ls -lrth /proc/$brick_pid/fd | grep -iw $L1 | grep -v ".glusterfs" | wc -l`
+TEST [ $n -eq 1 ]
+n=`ls -lrth /proc/$brick_pid/fd | grep -iw $L2 | grep -v ".glusterfs" | wc -l`
+TEST [ $n -eq 1 ]
+n=`ls -lrth /proc/$brick_pid/fd | grep -iw $L3 | grep -v ".glusterfs" | wc -l`
+TEST [ $n -eq 0 ]
+
+# Mount the brick root
+TEST mount -t xfs -o nouuid /dev/test_vg_3/brick_lvm $brick_root
+
+# Replace brick_pid file to test brick_attach code
+TEST cp $b1_pid_file $b3_pid_file
+
+# Start the volume all brick should be up
+TEST $CLI volume start $V0 force
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_up_bricks
+EXPECT 1 count_brick_processes
+
+# Make sure every brick root should be consumed by a brick process
+n=`ls -lrth /proc/$brick_pid/fd | grep -iw $L1 | grep -v ".glusterfs" | wc -l`
+TEST [ $n -eq 1 ]
+n=`ls -lrth /proc/$brick_pid/fd | grep -iw $L2 | grep -v ".glusterfs" | wc -l`
+TEST [ $n -eq 1 ]
+n=`ls -lrth /proc/$brick_pid/fd | grep -iw $L3 | grep -v ".glusterfs" | wc -l`
+TEST [ $n -eq 1 ]
+
+cleanup
diff --git a/tests/basic/posix/shared-statfs.t b/tests/basic/posix/shared-statfs.t
index 8caa9fa..3343956 100644
--- a/tests/basic/posix/shared-statfs.t
+++ b/tests/basic/posix/shared-statfs.t
@@ -23,6 +23,7 @@ TEST MOUNT_LOOP $LO2 $B0/${V0}2
# Create a subdir in mountpoint and use that for volume.
TEST $CLI volume create $V0 $H0:$B0/${V0}1/1 $H0:$B0/${V0}2/1;
TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" online_brick_count
TEST $GFS --volfile-server=$H0 --volfile-id=$V0 $M0
total_space=$(df -P $M0 | tail -1 | awk '{ print $2}')
# Keeping the size less than 200M mainly because XFS will use
@@ -38,6 +39,7 @@ EXPECT 'Stopped' volinfo_field $V0 'Status';
TEST $CLI volume add-brick $V0 $H0:$B0/${V0}1/2 $H0:$B0/${V0}2/2 $H0:$B0/${V0}1/3 $H0:$B0/${V0}2/3
TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "6" online_brick_count
TEST $GFS --volfile-server=$H0 --volfile-id=$V0 $M0
total_space=$(df -P $M0 | tail -1 | awk '{ print $2}')
TEST [ $total_space -gt 194000 -a $total_space -lt 200000 ]
diff --git a/tests/bitrot/bug-1373520.t b/tests/bitrot/bug-1373520.t
index 225d3b1..c09d424 100644
--- a/tests/bitrot/bug-1373520.t
+++ b/tests/bitrot/bug-1373520.t
@@ -11,6 +11,7 @@ TEST pidof glusterd
#Create a disperse volume
TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5}
TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "6" online_brick_count
EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status'
#Disable md-cache
diff --git a/tests/bugs/distribute/bug-1368012.t b/tests/bugs/distribute/bug-1368012.t
index f89314b..b861554 100644
--- a/tests/bugs/distribute/bug-1368012.t
+++ b/tests/bugs/distribute/bug-1368012.t
@@ -22,6 +22,7 @@ EXPECT "$V0" volinfo_field $V0 'Volume Name';
EXPECT 'Created' volinfo_field $V0 'Status';
## Start volume and verify
TEST $CLI volume start $V0;
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" online_brick_count
TEST $CLI volume set $V0 performance.stat-prefetch off
EXPECT 'Started' volinfo_field $V0 'Status';
TEST glusterfs -s $H0 --volfile-id=$V0 $M0
@@ -36,6 +37,7 @@ TEST permission_root=`stat -c "%A" $M0`
TEST echo $permission_root
#Add-brick
TEST $CLI volume add-brick $V0 $H0:/${V0}3
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "3" online_brick_count
#Allow one lookup to happen
TEST pushd $M0
diff --git a/tests/bugs/distribute/bug-853258.t b/tests/bugs/distribute/bug-853258.t
index e39f507..6817d9e 100755
--- a/tests/bugs/distribute/bug-853258.t
+++ b/tests/bugs/distribute/bug-853258.t
@@ -31,6 +31,7 @@ done
# Expand the volume and force assignment of new ranges.
TEST $CLI volume add-brick $V0 $H0:$B0/${V0}3
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "4" online_brick_count
# Force assignment of initial ranges.
TEST $CLI volume rebalance $V0 fix-layout start
EXPECT_WITHIN $REBALANCE_TIMEOUT "fix-layout completed" fix-layout_status_field $V0
diff --git a/tests/bugs/quota/bug-1293601.t b/tests/bugs/quota/bug-1293601.t
index def4ef9..741758b 100644
--- a/tests/bugs/quota/bug-1293601.t
+++ b/tests/bugs/quota/bug-1293601.t
@@ -9,6 +9,7 @@ TEST glusterd
TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4}
TEST $CLI volume start $V0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "4" online_brick_count
TEST $CLI volume quota $V0 enable
TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0;
@@ -27,6 +28,6 @@ EXPECT_WITHIN $MARKER_UPDATE_TIMEOUT "1.0MB" quotausage "/"
TEST $CLI volume quota $V0 disable
TEST $CLI volume quota $V0 enable
-EXPECT_WITHIN 40 "1.0MB" quotausage "/"
+EXPECT_WITHIN 60 "1.0MB" quotausage "/"
cleanup;
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
index 304cef6..09e10bf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -2844,7 +2844,7 @@ glusterd_do_lvm_snapshot_remove (glusterd_volinfo_t *snap_vol,
GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_vol, brickinfo, priv);
if (gf_is_service_running (pidfile, &pid)) {
(void) send_attach_req (this, brickinfo->rpc,
- brickinfo->path, NULL,
+ brickinfo->path, NULL, NULL,
GLUSTERD_BRICK_TERMINATE);
brickinfo->status = GF_BRICK_STOPPED;
}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 95df889..fe9cc75 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -2186,7 +2186,7 @@ retry:
goto out;
}
- ret = glusterd_brick_process_add_brick (brickinfo, volinfo);
+ ret = glusterd_brick_process_add_brick (brickinfo);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
GD_MSG_BRICKPROC_ADD_BRICK_FAILED, "Adding brick %s:%s "
@@ -2372,8 +2372,7 @@ out:
}
int
-glusterd_brick_process_add_brick (glusterd_brickinfo_t *brickinfo,
- glusterd_volinfo_t *volinfo)
+glusterd_brick_process_add_brick (glusterd_brickinfo_t *brickinfo)
{
int ret = -1;
xlator_t *this = NULL;
@@ -2500,7 +2499,7 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t *volinfo,
brickinfo->hostname, brickinfo->path);
(void) send_attach_req (this, brickinfo->rpc,
- brickinfo->path, NULL,
+ brickinfo->path, NULL, NULL,
GLUSTERD_BRICK_TERMINATE);
} else {
gf_msg_debug (this->name, 0, "About to stop glusterfsd"
@@ -5426,23 +5425,92 @@ static int32_t
attach_brick_callback (struct rpc_req *req, struct iovec *iov, int count,
void *v_frame)
{
- call_frame_t *frame = v_frame;
- glusterd_conf_t *conf = frame->this->private;
- glusterd_brickinfo_t *brickinfo = frame->local;
+ call_frame_t *frame = v_frame;
+ glusterd_conf_t *conf = frame->this->private;
+ glusterd_brickinfo_t *brickinfo = frame->local;
+ glusterd_brickinfo_t *other_brick = frame->cookie;
+ glusterd_volinfo_t *volinfo = NULL;
+ xlator_t *this = THIS;
+ int ret = -1;
+ char pidfile1[PATH_MAX] = {0};
+ char pidfile2[PATH_MAX] = {0};
+ gf_getspec_rsp rsp = {0,};
frame->local = NULL;
- brickinfo->port_registered = _gf_true;
+ frame->cookie = NULL;
+
+ ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+ if (ret < 0) {
+ gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+ ret = -1;
+ goto out;
+ }
+
+ ret = glusterd_get_volinfo_from_brick (other_brick->path,
+ &volinfo);
+ if (ret) {
+ gf_msg (THIS->name, GF_LOG_ERROR, 0,
+ GD_MSG_VOLINFO_GET_FAIL, "Failed to get volinfo"
+ " from brick(%s) so pidfile copying/unlink will fail",
+ other_brick->path);
+ goto out;
+ }
+ GLUSTERD_GET_BRICK_PIDFILE (pidfile1, volinfo, other_brick, conf);
+ volinfo = NULL;
+
+ ret = glusterd_get_volinfo_from_brick (brickinfo->path,
+ &volinfo);
+ if (ret) {
+ gf_msg (THIS->name, GF_LOG_ERROR, 0,
+ GD_MSG_VOLINFO_GET_FAIL, "Failed to get volinfo"
+ " from brick(%s) so pidfile copying/unlink will fail",
+ brickinfo->path);
+ goto out;
+ }
+ GLUSTERD_GET_BRICK_PIDFILE (pidfile2, volinfo, brickinfo, conf);
+
+ if (rsp.op_ret == 0) {
+ brickinfo->port_registered = _gf_true;
+
+ /* PID file is copied once brick has attached
+ successfully
+ */
+ glusterd_copy_file (pidfile1, pidfile2);
+ brickinfo->status = GF_BRICK_STARTED;
+ brickinfo->rpc = rpc_clnt_ref (other_brick->rpc);
+ gf_log (THIS->name, GF_LOG_INFO, "brick %s is attached successfully",
+ brickinfo->path);
+ } else {
+ gf_log (THIS->name, GF_LOG_INFO, "attach_brick failed pidfile"
+ " is %s for brick_path %s", pidfile2, brickinfo->path);
+ brickinfo->port = 0;
+ brickinfo->status = GF_BRICK_STOPPED;
+ ret = glusterd_brick_process_remove_brick (brickinfo);
+ if (ret)
+ gf_msg_debug (this->name, 0, "Couldn't remove brick from"
+ " brick process");
+ LOCK (&volinfo->lock);
+ ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
+ UNLOCK (&volinfo->lock);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_VOLINFO_SET_FAIL,
+ "Failed to store volinfo of "
+ "%s volume", volinfo->volname);
+ goto out;
+ }
+ }
+out:
synclock_lock (&conf->big_lock);
--(conf->blockers);
synclock_unlock (&conf->big_lock);
-
STACK_DESTROY (frame->root);
return 0;
}
int
send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path,
- glusterd_brickinfo_t *brickinfo, int op)
+ glusterd_brickinfo_t *brickinfo, glusterd_brickinfo_t *other_brick, int op)
{
int ret = -1;
struct iobuf *iobuf = NULL;
@@ -5516,6 +5584,7 @@ send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path,
if (op == GLUSTERD_BRICK_ATTACH) {
frame->local = brickinfo;
+ frame->cookie = other_brick;
cbkfn = attach_brick_callback;
}
/* Send the msg */
@@ -5582,27 +5651,19 @@ attach_brick (xlator_t *this,
rpc = rpc_clnt_ref (other_brick->rpc);
if (rpc) {
ret = send_attach_req (this, rpc, path, brickinfo,
+ other_brick,
GLUSTERD_BRICK_ATTACH);
rpc_clnt_unref (rpc);
if (!ret) {
ret = pmap_registry_extend (this, other_brick->port,
- brickinfo->path);
+ brickinfo->path);
if (ret != 0) {
gf_log (this->name, GF_LOG_ERROR,
"adding brick to process failed");
- return ret;
+ goto out;
}
-
- /* PID file is copied once brick has attached
- successfully
- */
- glusterd_copy_file (pidfile1, pidfile2);
brickinfo->port = other_brick->port;
- brickinfo->status = GF_BRICK_STARTED;
- brickinfo->rpc =
- rpc_clnt_ref (other_brick->rpc);
- ret = glusterd_brick_process_add_brick (brickinfo,
- volinfo);
+ ret = glusterd_brick_process_add_brick (brickinfo);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
GD_MSG_BRICKPROC_ADD_BRICK_FAILED,
@@ -5611,29 +5672,23 @@ attach_brick (xlator_t *this,
brickinfo->path);
return ret;
}
-
- if (ret) {
- gf_msg_debug (this->name, 0, "Add brick"
- " to brick process failed");
- return ret;
- }
-
return 0;
}
}
/*
- * It might not actually be safe to manipulate the lock like
- * this, but if we don't then the connection can never actually
- * complete and retries are useless. Unfortunately, all of the
- * alternatives (e.g. doing all of this in a separate thread)
- * are much more complicated and risky. TBD: see if there's a
- * better way
+ * It might not actually be safe to manipulate the lock
+ * like this, but if we don't then the connection can
+ * never actually complete and retries are useless.
+ * Unfortunately, all of the alternatives (e.g. doing
+ * all of this in a separate thread) are much more
+ * complicated and risky.
+ * TBD: see if there's a better way
*/
synclock_unlock (&conf->big_lock);
sleep (1);
synclock_lock (&conf->big_lock);
}
-
+out:
gf_log (this->name, GF_LOG_WARNING,
"attach failed for %s", brickinfo->path);
return ret;
@@ -5855,6 +5910,7 @@ find_compatible_brick (glusterd_conf_t *conf,
return NULL;
}
+
/* Below function is use to populate sockpath based on passed pid
value as a argument after check the value from proc and also
check if passed pid is match with running glusterfs process
@@ -5941,6 +5997,62 @@ glusterd_get_sock_from_brick_pid (int pid, char *sockpath, size_t len)
}
+char *
+search_brick_path_from_proc (pid_t brick_pid, char *brickpath)
+{
+ struct dirent *dp = NULL;
+ DIR *dirp = NULL;
+ size_t len = 0;
+ int fd = -1;
+ char path[PATH_MAX] = {0,};
+ char sym[PATH_MAX] = {0,};
+ struct dirent scratch[2] = {{0,},};
+ char *brick_path = NULL;
+
+ if (!brickpath)
+ goto out;
+
+ sprintf(path, "/proc/%d/fd/", brick_pid);
+ dirp = sys_opendir (path);
+ if (!dirp)
+ goto out;
+
+ len = strlen (path);
+ if (len >= (sizeof(path) - 2))
+ goto out;
+
+ fd = dirfd (dirp);
+ if (fd < 0)
+ goto out;
+
+ memset(path, 0, sizeof(path));
+ memset(sym, 0, sizeof(sym));
+
+ while ((dp = sys_readdir(dirp, scratch))) {
+ if (!strcmp(dp->d_name, ".") ||
+ !strcmp(dp->d_name, ".."))
+ continue;
+
+ /* check for non numerical descriptors */
+ if (!strtol(dp->d_name, (char **)NULL, 10))
+ continue;
+
+ len = readlinkat (fd, dp->d_name, sym, sizeof(sym) - 1);
+ if (len > 1) {
+ sym[len] = '\0';
+ if (!strcmp (sym, brickpath)) {
+ brick_path = gf_strdup(sym);
+ break;
+ }
+ memset (sym, 0, sizeof (sym));
+ }
+ }
+out:
+ sys_closedir(dirp);
+ return brick_path;
+}
+
+
int
glusterd_brick_start (glusterd_volinfo_t *volinfo,
glusterd_brickinfo_t *brickinfo,
@@ -5954,7 +6066,9 @@ glusterd_brick_start (glusterd_volinfo_t *volinfo,
int32_t pid = -1;
char pidfile[PATH_MAX] = {0};
char socketpath[PATH_MAX] = {0};
+ char *brickpath = NULL;
glusterd_volinfo_t *other_vol;
+ struct statvfs brickstat = {0,};
this = THIS;
GF_ASSERT (this);
@@ -6000,6 +6114,28 @@ glusterd_brick_start (glusterd_volinfo_t *volinfo,
brickinfo->start_triggered = _gf_true;
GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, conf);
+
+ ret = sys_statvfs (brickinfo->path, &brickstat);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR,
+ errno, GD_MSG_BRICKINFO_CREATE_FAIL,
+ "failed to get statfs() call on brick %s",
+ brickinfo->path);
+ goto out;
+ }
+
+ /* Compare fsid is helpful to ensure the existence of a brick_root
+ path before the start/attach a brick
+ */
+ if (brickinfo->statfs_fsid &&
+ (brickinfo->statfs_fsid != brickstat.f_fsid)) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "fsid comparison is failed it means Brick root path"
+ " %s is not created by glusterd, start/attach will also fail",
+ brickinfo->path);
+ goto out;
+ }
+
if (gf_is_service_running (pidfile, &pid)) {
if (brickinfo->status != GF_BRICK_STARTING &&
brickinfo->status != GF_BRICK_STARTED) {
@@ -6019,12 +6155,29 @@ glusterd_brick_start (glusterd_volinfo_t *volinfo,
* TBD: re-use RPC connection across bricks
*/
if (is_brick_mx_enabled ()) {
+ brickpath = search_brick_path_from_proc (pid, brickinfo->path);
+ if (!brickpath) {
+ gf_log (this->name, GF_LOG_INFO,
+ "Either pid %d is not running or brick"
+ " path %s is not consumed so cleanup pidfile",
+ pid, brickinfo->path);
+ /* search brick is failed so unlink pidfile */
+ if (sys_access (pidfile , R_OK) == 0) {
+ sys_unlink (pidfile);
+ }
+ goto run;
+ }
+ GF_FREE (brickpath);
ret = glusterd_get_sock_from_brick_pid (pid, socketpath,
sizeof(socketpath));
if (ret) {
- gf_log (this->name, GF_LOG_DEBUG,
+ gf_log (this->name, GF_LOG_INFO,
"Either pid %d is not running or is not match"
" with any running brick process ", pid);
+ /* Fetch unix socket is failed so unlink pidfile */
+ if (sys_access (pidfile , R_OK) == 0) {
+ sys_unlink (pidfile);
+ }
goto run;
}
} else {
@@ -6039,7 +6192,7 @@ glusterd_brick_start (glusterd_volinfo_t *volinfo,
(void) glusterd_brick_connect (volinfo, brickinfo,
socketpath);
- ret = glusterd_brick_process_add_brick (brickinfo, volinfo);
+ ret = glusterd_brick_process_add_brick (brickinfo);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
GD_MSG_BRICKPROC_ADD_BRICK_FAILED,
@@ -6079,6 +6232,10 @@ run:
if (ret == 0) {
goto out;
}
+ /* Attach_brick is failed so unlink pidfile */
+ if (sys_access (pidfile , R_OK) == 0) {
+ sys_unlink (pidfile);
+ }
}
/*
@@ -7063,14 +7220,15 @@ glusterd_add_brick_to_dict (glusterd_volinfo_t *volinfo,
dict_t *dict, int32_t count)
{
- int ret = -1;
- int32_t pid = -1;
- char key[1024] = {0};
- char base_key[1024] = {0};
- char pidfile[PATH_MAX] = {0};
+ int ret = -1;
+ int32_t pid = -1;
+ char key[1024] = {0};
+ char base_key[1024] = {0};
+ char pidfile[PATH_MAX] = {0};
xlator_t *this = NULL;
glusterd_conf_t *priv = NULL;
- gf_boolean_t brick_online = _gf_false;
+ gf_boolean_t brick_online = _gf_false;
+ char *brickpath = NULL;
GF_ASSERT (volinfo);
GF_ASSERT (brickinfo);
@@ -7127,7 +7285,20 @@ glusterd_add_brick_to_dict (glusterd_volinfo_t *volinfo,
if (glusterd_is_brick_started (brickinfo)) {
if (gf_is_service_running (pidfile, &pid) &&
brickinfo->port_registered) {
- brick_online = _gf_true;
+ if (!is_brick_mx_enabled ()) {
+ brick_online = _gf_true;
+ } else {
+ brickpath = search_brick_path_from_proc (pid, brickinfo->path);
+ if (!brickpath) {
+ gf_log (this->name, GF_LOG_INFO,
+ "brick path %s is not consumed",
+ brickinfo->path);
+ brick_online = _gf_false;
+ } else {
+ brick_online = _gf_true;
+ GF_FREE (brickpath);
+ }
+ }
} else {
pid = -1;
}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 4c9561e..4835728 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -179,8 +179,7 @@ int32_t
glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo);
int
-glusterd_brick_process_add_brick (glusterd_brickinfo_t *brickinfo,
- glusterd_volinfo_t *volinfo);
+glusterd_brick_process_add_brick (glusterd_brickinfo_t *brickinfo);
int
glusterd_brick_process_remove_brick (glusterd_brickinfo_t *brickinfo);
@@ -200,7 +199,8 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t *volinfo,
int
send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path,
- glusterd_brickinfo_t *brick, int op);
+ glusterd_brickinfo_t *brick,
+ glusterd_brickinfo_t *other_brick, int op);
glusterd_volinfo_t *
glusterd_volinfo_ref (glusterd_volinfo_t *volinfo);
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index e34d58a..8bb0b6d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2585,8 +2585,13 @@ glusterd_start_volume (glusterd_volinfo_t *volinfo, int flags,
}
glusterd_set_volume_status (volinfo, GLUSTERD_STATUS_STARTED);
-
+ /* Update volinfo on disk in critical section because
+ attach_brick_callback can also call store_volinfo for same
+ volume to update volinfo on disk
+ */
+ LOCK (&volinfo->lock);
ret = glusterd_store_volinfo (volinfo, verincrement);
+ UNLOCK (&volinfo->lock);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
GD_MSG_VOLINFO_SET_FAIL,
--
1.8.3.1