From 15d48e766095a3ade29f82a6d1a26c65ef6c97d4 Mon Sep 17 00:00:00 2001 From: Pranith Kumar K Date: Tue, 3 Jul 2018 14:14:59 +0530 Subject: [PATCH 685/685] glusterd: show brick online after port registration even in brick-mux Upstream patch: https://review.gluster.org/20451 Problem: With brick-mux even before brick attach is complete on the bricks glusterd marks them as online. This can lead to a race where scripts that check if the bricks are online to assume that the brick is online before it is completely online. Fix: Wait for the callback from the brick before marking the port as registered so that volume status will show the correct status of the brick. BUG: 1598353 >fixes bz#1597568 Change-Id: Icd3dc62506af0cf75195e96746695db823312051 Signed-off-by: Pranith Kumar K Reviewed-on: https://code.engineering.redhat.com/gerrit/143269 Reviewed-by: Atin Mukherjee Tested-by: RHGS Build Bot --- xlators/mgmt/glusterd/src/glusterd-snapshot.c | 2 +- xlators/mgmt/glusterd/src/glusterd-utils.c | 36 +++++++++++++++++++++------ xlators/mgmt/glusterd/src/glusterd-utils.h | 3 ++- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c index 639282e..497bdba 100644 --- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c +++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c @@ -2827,7 +2827,7 @@ glusterd_do_lvm_snapshot_remove (glusterd_volinfo_t *snap_vol, GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_vol, brickinfo, priv); if (gf_is_service_running (pidfile, &pid)) { (void) send_attach_req (this, brickinfo->rpc, - brickinfo->path, + brickinfo->path, NULL, GLUSTERD_BRICK_TERMINATE); brickinfo->status = GF_BRICK_STOPPED; } diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index 828155d..bbf6f7d 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -92,9 +92,6 @@ #define NLMV4_VERSION 4 #define NLMV1_VERSION 1 -int -send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path, int op); - gf_boolean_t is_brick_mx_enabled (void) { @@ -2409,7 +2406,7 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t *volinfo, brickinfo->hostname, brickinfo->path); (void) send_attach_req (this, brickinfo->rpc, - brickinfo->path, + brickinfo->path, NULL, GLUSTERD_BRICK_TERMINATE); } else { gf_msg_debug (this->name, 0, "About to stop glusterfsd" @@ -5330,8 +5327,27 @@ my_callback (struct rpc_req *req, struct iovec *iov, int count, void *v_frame) return 0; } +static int32_t +attach_brick_callback (struct rpc_req *req, struct iovec *iov, int count, + void *v_frame) +{ + call_frame_t *frame = v_frame; + glusterd_conf_t *conf = frame->this->private; + glusterd_brickinfo_t *brickinfo = frame->local; + + frame->local = NULL; + brickinfo->port_registered = _gf_true; + synclock_lock (&conf->big_lock); + --(conf->blockers); + synclock_unlock (&conf->big_lock); + + STACK_DESTROY (frame->root); + return 0; +} + int -send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path, int op) +send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path, + glusterd_brickinfo_t *brickinfo, int op) { int ret = -1; struct iobuf *iobuf = NULL; @@ -5345,6 +5361,7 @@ send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path, int op) struct rpc_clnt_connection *conn; glusterd_conf_t *conf = this->private; extern struct rpc_clnt_program gd_brick_prog; + fop_cbk_fn_t cbkfn = my_callback; if (!rpc) { gf_log (this->name, GF_LOG_ERROR, "called with null rpc"); @@ -5402,10 +5419,14 @@ send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path, int op) iov.iov_len = ret; + if (op == GLUSTERD_BRICK_ATTACH) { + frame->local = brickinfo; + cbkfn = attach_brick_callback; + } /* Send the msg */ ++(conf->blockers); ret = rpc_clnt_submit (rpc, &gd_brick_prog, op, - my_callback, &iov, 1, NULL, 0, iobref, + cbkfn, &iov, 1, NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL); return ret; @@ -5465,7 +5486,7 @@ attach_brick (xlator_t *this, for (tries = 15; tries > 0; --tries) { rpc = rpc_clnt_ref (other_brick->rpc); if (rpc) { - ret = send_attach_req (this, rpc, path, + ret = send_attach_req (this, rpc, path, brickinfo, GLUSTERD_BRICK_ATTACH); rpc_clnt_unref (rpc); if (!ret) { @@ -5485,7 +5506,6 @@ attach_brick (xlator_t *this, brickinfo->status = GF_BRICK_STARTED; brickinfo->rpc = rpc_clnt_ref (other_brick->rpc); - brickinfo->port_registered = _gf_true; ret = glusterd_brick_process_add_brick (brickinfo, volinfo); if (ret) { diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h index 259088b..d1156dc 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-utils.h @@ -197,7 +197,8 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t *volinfo, gf_boolean_t del_brick); int -send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path, int op); +send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path, + glusterd_brickinfo_t *brick, int op); glusterd_volinfo_t * glusterd_volinfo_ref (glusterd_volinfo_t *volinfo); -- 1.8.3.1