From f58abec63fb325e0e1c21fe3fe127de2e4a85d7d Mon Sep 17 00:00:00 2001 From: Mohit Agrawal Date: Thu, 24 Jan 2019 18:45:54 +0530 Subject: [PATCH 509/510] core: heketi-cli is throwing error "target is busy" Problem: At the time of deleting block hosting volume through heketi-cli , it is throwing an error "target is busy". cli is throwing an error because brick is not detached successfully and brick is not detached due to race condition to cleanp xprt associated with detached brick Solution: To avoid xprt specifc race condition introduce an atomic flag on rpc_transport > Change-Id: Id4ff1fe8375a63be71fb3343f455190a1b8bb6d4 > fixes: bz#1668190 > (Cherry pick from commit 04f84756e1baa5eff4560339700f82970eaa5d80) > (Reviewed on upstream link https://review.gluster.org/#/c/glusterfs/+/22073/) Change-Id: Ie3786b569ee03569bc3ac970925732dd834a76dc BUG: 1669020 Signed-off-by: Mohit Agrawal Reviewed-on: https://code.engineering.redhat.com/gerrit/161388 Tested-by: RHGS Build Bot Reviewed-by: Sunil Kumar Heggodu Gopala Acharya --- rpc/rpc-lib/src/rpc-transport.c | 1 + rpc/rpc-lib/src/rpc-transport.h | 1 + xlators/protocol/server/src/server.c | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c index 77abf9617..0c6ab6694 100644 --- a/rpc/rpc-lib/src/rpc-transport.c +++ b/rpc/rpc-lib/src/rpc-transport.c @@ -371,6 +371,7 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name) } INIT_LIST_HEAD (&trans->list); + GF_ATOMIC_INIT(trans->disconnect_progress, 0); return_trans = trans; diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h index 23246c564..f5fb6e13b 100644 --- a/rpc/rpc-lib/src/rpc-transport.h +++ b/rpc/rpc-lib/src/rpc-transport.h @@ -217,6 +217,7 @@ struct rpc_transport { * layer or in client management notification handler functions */ gf_boolean_t connect_failed; + gf_atomic_t disconnect_progress; }; struct rpc_transport_ops { diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c index 104615265..ba3b8316d 100644 --- a/xlators/protocol/server/src/server.c +++ b/xlators/protocol/server/src/server.c @@ -553,6 +553,11 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event, break; } + /* Set the disconnect_progress flag to 1 to avoid races + during brick detach while brick mux is enabled + */ + GF_ATOMIC_INIT(trans->disconnect_progress, 1); + /* transport has to be removed from the list upon disconnect * irrespective of whether lock self heal is off or on, since * new transport will be created upon reconnect. @@ -1638,6 +1643,7 @@ notify (xlator_t *this, int32_t event, void *data, ...) glusterfs_ctx_t *ctx = NULL; gf_boolean_t xprt_found = _gf_false; uint64_t totxprt = 0; + uint64_t totdisconnect = 0; GF_VALIDATE_OR_GOTO (THIS->name, this, out); conf = this->private; @@ -1715,6 +1721,10 @@ notify (xlator_t *this, int32_t event, void *data, ...) if (!xprt->xl_private) { continue; } + + if (GF_ATOMIC_GET(xprt->disconnect_progress)) + continue; + if (xprt->xl_private->bound_xl == data) { totxprt++; } @@ -1740,15 +1750,23 @@ notify (xlator_t *this, int32_t event, void *data, ...) if (!xprt->xl_private) { continue; } + + if (GF_ATOMIC_GET(xprt->disconnect_progress)) + continue; + if (xprt->xl_private->bound_xl == data) { gf_log (this->name, GF_LOG_INFO, "disconnecting %s", xprt->peerinfo.identifier); xprt_found = _gf_true; + totdisconnect++; rpc_transport_disconnect (xprt, _gf_false); } } + if (totxprt > totdisconnect) + GF_ATOMIC_SUB(victim->xprtrefcnt, (totxprt - totdisconnect)); + pthread_mutex_unlock (&conf->mutex); if (this->ctx->active) { top = this->ctx->active->first; -- 2.20.1