From 9b26837434977601f1e48477904486ea032f742a Mon Sep 17 00:00:00 2001

From: Mohit Agrawal <moagrawa@redhat.com>

Date: Mon, 8 Oct 2018 22:32:28 +0530

Subject: [PATCH 390/399] core: glusterfsd keeping fd open in index xlator

Problem: Current resource cleanup sequence is not

         perfect while brick mux is enabled

Solution: 1. Destroying xprt after cleanup all fd associated

             with a client

          2. Before call fini for brick xlators ensure no stub

             should be running on a brick

> Change-Id: I86195785e428f57d3ef0da3e4061021fafacd435

> fixes: bz#1631357

> (cherry picked from commit 5bc4594dabc08fd4de1940c044946e33037f2ac7)

> (Reviewed on upstream link https://review.gluster.org/#/c/glusterfs/+/21235/)

Change-Id: I0676a2f8e42557c1107a877406e255b93a77a269

BUG: 1631372

Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/152170

Tested-by: RHGS Build Bot <nigelb@redhat.com>

Reviewed-by: Raghavendra Gowdappa <rgowdapp@redhat.com>

Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>

---

 libglusterfs/src/defaults-tmpl.c                |   8 +-

 libglusterfs/src/xlator.c                       |   2 +

 libglusterfs/src/xlator.h                       |   7 +

 xlators/features/index/src/index.c              |  50 ++++++-

 xlators/features/index/src/index.h              |   2 +

 xlators/performance/io-threads/src/io-threads.c |  45 ++++++-

 xlators/performance/io-threads/src/io-threads.h |   2 +-

 xlators/protocol/server/src/server-handshake.c  |  23 +++-

 xlators/protocol/server/src/server-helpers.c    |  79 +++++++++--

 xlators/protocol/server/src/server-helpers.h    |   2 +-

 xlators/protocol/server/src/server.c            | 171 ++++++++++++++++--------

 xlators/protocol/server/src/server.h            |   9 +-

 xlators/storage/posix/src/posix.c               |  11 ++

 13 files changed, 333 insertions(+), 78 deletions(-)

diff --git a/libglusterfs/src/defaults-tmpl.c b/libglusterfs/src/defaults-tmpl.c

index 0ef14d5..3fdeabb 100644

--- a/libglusterfs/src/defaults-tmpl.c

+++ b/libglusterfs/src/defaults-tmpl.c

@@ -119,6 +119,8 @@ default_release (xlator_t *this, fd_t *fd)

 int

 default_notify (xlator_t *this, int32_t event, void *data, ...)

 {

+        xlator_t *victim = data;

+

         switch (event) {

         case GF_EVENT_PARENT_UP:

         case GF_EVENT_PARENT_DOWN:

@@ -126,7 +128,11 @@ default_notify (xlator_t *this, int32_t event, void *data, ...)

                 xlator_list_t *list = this->children;

                 while (list) {

-                        xlator_notify (list->xlator, event, this);

+                        if (victim && victim->cleanup_starting) {

+                                xlator_notify(list->xlator, event, victim);

+                        } else {

+                                xlator_notify(list->xlator, event, this);

+                        }

                         list = list->next;

                 }

         }

diff --git a/libglusterfs/src/xlator.c b/libglusterfs/src/xlator.c

index 1cf4a63..8aa8aa1 100644

--- a/libglusterfs/src/xlator.c

+++ b/libglusterfs/src/xlator.c

@@ -489,6 +489,8 @@ xlator_init (xlator_t *xl)

                 xl->mem_acct_init (xl);

         xl->instance_name = NULL;

+        GF_ATOMIC_INIT(xl->xprtrefcnt, 0);

+        GF_ATOMIC_INIT(xl->fd_cnt, 0);

         if (!xl->init) {

                 gf_msg (xl->name, GF_LOG_WARNING, 0, LG_MSG_INIT_FAILED,

                         "No init() found");

diff --git a/libglusterfs/src/xlator.h b/libglusterfs/src/xlator.h

index 7434da8..1879641 100644

--- a/libglusterfs/src/xlator.h

+++ b/libglusterfs/src/xlator.h

@@ -965,7 +965,14 @@ struct _xlator {

         /* flag to avoid recall of xlator_mem_cleanup for xame xlator */

         uint32_t call_cleanup;

+        /* Variable to save fd_count for detach brick */

+        gf_atomic_t fd_cnt;

+        /* Variable to save xprt associated for detach brick */

+        gf_atomic_t xprtrefcnt;

+

+        /* Flag to notify got CHILD_DOWN event for detach brick */

+        uint32_t notify_down;

 };

 typedef struct {

diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c

index f3b0270..bf3f4dd 100644

--- a/xlators/features/index/src/index.c

+++ b/xlators/features/index/src/index.c

@@ -188,6 +188,7 @@ worker_enqueue (xlator_t *this, call_stub_t *stub)

         pthread_mutex_lock (&priv->mutex);

         {

                 __index_enqueue (&priv->callstubs, stub);

+                GF_ATOMIC_INC(priv->stub_cnt);

                 pthread_cond_signal (&priv->cond);

         }

         pthread_mutex_unlock (&priv->mutex);

@@ -223,11 +224,18 @@ index_worker (void *data)

                         }

                         if (!bye)

                                 stub = __index_dequeue (&priv->callstubs);

+                        if (bye) {

+                                priv->curr_count--;

+                                if (priv->curr_count == 0)

+                                        pthread_cond_broadcast(&priv->cond);

+                        }

                 }

                 pthread_mutex_unlock (&priv->mutex);

-                if (stub) /* guard against spurious wakeups */

+                if (stub) {/* guard against spurious wakeups */

                         call_resume (stub);

+                        GF_ATOMIC_DEC(priv->stub_cnt);

+                }

                 stub = NULL;

                 if (bye)

                         break;

@@ -2375,6 +2383,7 @@ init (xlator_t *this)

                 gf_uuid_generate (priv->internal_vgfid[i]);

         INIT_LIST_HEAD (&priv->callstubs);

+        GF_ATOMIC_INIT(priv->stub_cnt, 0);

         this->local_pool = mem_pool_new (index_local_t, 64);

         if (!this->local_pool) {

@@ -2403,6 +2412,7 @@ init (xlator_t *this)

         index_set_link_count (priv, count, XATTROP);

         priv->down = _gf_false;

+        priv->curr_count = 0;

         ret = gf_thread_create (&priv->thread, &w_attr, index_worker, this,

                                 "idxwrker");

         if (ret) {

@@ -2411,7 +2421,7 @@ init (xlator_t *this)

                         "Failed to create worker thread, aborting");

                 goto out;

         }

-

+        priv->curr_count++;

         ret = 0;

 out:

         if (ret) {

@@ -2528,6 +2538,9 @@ notify (xlator_t *this, int event, void *data, ...)

 {

         int     ret = 0;

         index_priv_t *priv = NULL;

+        uint64_t stub_cnt = 0;

+        xlator_t *victim = data;

+        struct timespec sleep_till = {0,};

         if (!this)

                 return 0;

@@ -2536,6 +2549,39 @@ notify (xlator_t *this, int event, void *data, ...)

         if (!priv)

                 return 0;

+        if ((event == GF_EVENT_PARENT_DOWN) && victim->cleanup_starting) {

+                stub_cnt = GF_ATOMIC_GET(priv->stub_cnt);

+                clock_gettime(CLOCK_REALTIME, &sleep_till);

+                sleep_till.tv_sec += 1;

+

+                /* Wait for draining stub from queue before notify PARENT_DOWN */

+                pthread_mutex_lock(&priv->mutex);

+                {

+                        while (stub_cnt) {

+                                (void)pthread_cond_timedwait(&priv->cond, &priv->mutex,

+                                                             &sleep_till);

+                                stub_cnt = GF_ATOMIC_GET(priv->stub_cnt);

+                        }

+                }

+                pthread_mutex_unlock(&priv->mutex);

+                gf_log(this->name, GF_LOG_INFO,

+                       "Notify GF_EVENT_PARENT_DOWN for brick %s", victim->name);

+        }

+

+        if ((event == GF_EVENT_CHILD_DOWN) && victim->cleanup_starting) {

+                pthread_mutex_lock(&priv->mutex);

+                {

+                        priv->down = _gf_true;

+                        pthread_cond_broadcast(&priv->cond);

+                        while (priv->curr_count)

+                                pthread_cond_wait(&priv->cond, &priv->mutex);

+                }

+                pthread_mutex_unlock(&priv->mutex);

+

+                gf_log(this->name, GF_LOG_INFO,

+                       "Notify GF_EVENT_CHILD_DOWN for brick %s", victim->name);

+        }

+

         ret = default_notify (this, event, data);

         return ret;

 }

diff --git a/xlators/features/index/src/index.h b/xlators/features/index/src/index.h

index ae9091d..d935294 100644

--- a/xlators/features/index/src/index.h

+++ b/xlators/features/index/src/index.h

@@ -62,6 +62,8 @@ typedef struct index_priv {

         int64_t  pending_count;

         pthread_t thread;

         gf_boolean_t down;

+        gf_atomic_t stub_cnt;

+        int32_t curr_count;

 } index_priv_t;

 typedef struct index_local {

diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c

index 1e1816a..5c47072 100644

--- a/xlators/performance/io-threads/src/io-threads.c

+++ b/xlators/performance/io-threads/src/io-threads.c

@@ -120,7 +120,7 @@ __iot_dequeue (iot_conf_t *conf, int *pri)

         if (!stub)

                 return NULL;

-        conf->queue_size--;

+        GF_ATOMIC_DEC(conf->queue_size);

         conf->queue_sizes[*pri]--;

         return stub;

@@ -153,7 +153,7 @@ __iot_enqueue (iot_conf_t *conf, call_stub_t *stub, int pri)

         }

         list_add_tail (&stub->list, &ctx->reqs);

-        conf->queue_size++;

+        GF_ATOMIC_INC(conf->queue_size);

         conf->queue_sizes[pri]++;

 }

@@ -182,7 +182,7 @@ iot_worker (void *data)

                                 conf->ac_iot_count[pri]--;

                                 pri = -1;

                         }

-                        while (conf->queue_size == 0) {

+                        while (GF_ATOMIC_GET(conf->queue_size) == 0) {

                                 if (conf->down) {

                                         bye = _gf_true;/*Avoid sleep*/

                                         break;

@@ -816,7 +816,7 @@ __iot_workers_scale (iot_conf_t *conf)

                         gf_msg_debug (conf->this->name, 0,

                                       "scaled threads to %d (queue_size=%d/%d)",

                                       conf->curr_count,

-                                      conf->queue_size, scale);

+                                      GF_ATOMIC_GET(conf->queue_size), scale);

                 } else {

                         break;

                 }

@@ -1030,6 +1030,7 @@ init (xlator_t *this)

                         bool, out);

         conf->this = this;

+        GF_ATOMIC_INIT(conf->queue_size, 0);

         for (i = 0; i < IOT_PRI_MAX; i++) {

                 INIT_LIST_HEAD (&conf->clients[i]);

@@ -1073,9 +1074,41 @@ int

 notify (xlator_t *this, int32_t event, void *data, ...)

 {

         iot_conf_t *conf = this->private;

+        xlator_t *victim = data;

+        uint64_t queue_size = 0;

+        struct timespec sleep_till = {0, };

+

+        if (GF_EVENT_PARENT_DOWN == event) {

+                if (victim->cleanup_starting) {

+                        clock_gettime(CLOCK_REALTIME, &sleep_till);

+                        sleep_till.tv_sec += 1;

+                        /* Wait for draining stub from queue before notify PARENT_DOWN */

+                        queue_size = GF_ATOMIC_GET(conf->queue_size);

+

+                        pthread_mutex_lock(&conf->mutex);

+                        {

+                                while (queue_size) {

+                                        (void)pthread_cond_timedwait(&conf->cond, &conf->mutex,

+                                                                     &sleep_till);

+                                        queue_size = GF_ATOMIC_GET(conf->queue_size);

+                                }

+                        }

+                        pthread_mutex_unlock(&conf->mutex);

-        if (GF_EVENT_PARENT_DOWN == event)

-                iot_exit_threads (conf);

+                        gf_log(this->name, GF_LOG_INFO,

+                               "Notify GF_EVENT_PARENT_DOWN for brick %s", victim->name);

+                } else {

+                        iot_exit_threads(conf);

+                }

+        }

+

+        if (GF_EVENT_CHILD_DOWN == event) {

+                if (victim->cleanup_starting) {

+                        iot_exit_threads(conf);

+                        gf_log(this->name, GF_LOG_INFO,

+                               "Notify GF_EVENT_CHILD_DOWN for brick %s", victim->name);

+                }

+        }

         default_notify (this, event, data);

diff --git a/xlators/performance/io-threads/src/io-threads.h b/xlators/performance/io-threads/src/io-threads.h

index 9648f74..7a6973c 100644

--- a/xlators/performance/io-threads/src/io-threads.h

+++ b/xlators/performance/io-threads/src/io-threads.h

@@ -75,7 +75,7 @@ struct iot_conf {

         int32_t              ac_iot_limit[IOT_PRI_MAX];

         int32_t              ac_iot_count[IOT_PRI_MAX];

         int                  queue_sizes[IOT_PRI_MAX];

-        int                  queue_size;

+        gf_atomic_t          queue_size;

         pthread_attr_t       w_attr;

         gf_boolean_t         least_priority; /*Enable/Disable least-priority */

diff --git a/xlators/protocol/server/src/server-handshake.c b/xlators/protocol/server/src/server-handshake.c

index 12f620c..75577fa 100644

--- a/xlators/protocol/server/src/server-handshake.c

+++ b/xlators/protocol/server/src/server-handshake.c

@@ -576,6 +576,7 @@ server_setvolume (rpcsvc_request_t *req)

                 goto fail;

         }

+        pthread_mutex_lock(&conf->mutex);

         list_for_each_entry (tmp, &conf->child_status->status_list,

                                                                   status_list) {

                 if (strcmp (tmp->name, name) == 0)

@@ -583,7 +584,7 @@ server_setvolume (rpcsvc_request_t *req)

         }

         if (!tmp->name) {

-                gf_msg (this->name, GF_LOG_ERROR, 0,

+                gf_msg (this->name, GF_LOG_INFO, 0,

                         PS_MSG_CHILD_STATUS_FAILED,

                         "No xlator %s is found in "

                         "child status list", name);

@@ -594,7 +595,21 @@ server_setvolume (rpcsvc_request_t *req)

                                 PS_MSG_DICT_GET_FAILED,

                                 "Failed to set 'child_up' for xlator %s "

                                 "in the reply dict", tmp->name);

+                if (!tmp->child_up) {

+                        ret = dict_set_str(reply, "ERROR",

+                                           "Not received child_up for this xlator");

+                        if (ret < 0)

+                                gf_msg_debug(this->name, 0, "failed to set error msg");

+

+                        gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_CHILD_STATUS_FAILED,

+                               "Not received child_up for this xlator %s", name);

+                        op_ret = -1;

+                        op_errno = EAGAIN;

+                        pthread_mutex_unlock(&conf->mutex);

+                        goto fail;

+                }

         }

+        pthread_mutex_unlock(&conf->mutex);

         ret = dict_get_str (params, "process-uuid", &client_uid);

         if (ret < 0) {

@@ -666,7 +681,7 @@ server_setvolume (rpcsvc_request_t *req)

         if (serv_ctx->lk_version != 0 &&

             serv_ctx->lk_version != lk_version) {

                 (void) server_connection_cleanup (this, client,

-                                                  INTERNAL_LOCKS | POSIX_LOCKS);

+                                                  INTERNAL_LOCKS | POSIX_LOCKS, NULL);

         }

         pthread_mutex_lock (&conf->mutex);

@@ -812,9 +827,9 @@ server_setvolume (rpcsvc_request_t *req)

                 req->trans->clnt_options = dict_ref(params);

                 gf_msg (this->name, GF_LOG_INFO, 0, PS_MSG_CLIENT_ACCEPTED,

-                        "accepted client from %s (version: %s)",

+                        "accepted client from %s (version: %s) with subvol %s",

                         client->client_uid,

-                        (clnt_version) ? clnt_version : "old");

+                        (clnt_version) ? clnt_version : "old", name);

                 gf_event (EVENT_CLIENT_CONNECT, "client_uid=%s;"

                           "client_identifier=%s;server_identifier=%s;"

diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c

index c492ab1..99256bf 100644

--- a/xlators/protocol/server/src/server-helpers.c

+++ b/xlators/protocol/server/src/server-helpers.c

@@ -242,16 +242,51 @@ server_connection_cleanup_flush_cbk (call_frame_t *frame, void *cookie,

         int32_t    ret    = -1;

         fd_t      *fd     = NULL;

         client_t  *client = NULL;

+        uint64_t fd_cnt = 0;

+        xlator_t *victim = NULL;

+        server_conf_t *conf = NULL;

+        xlator_t *serv_xl = NULL;

+        rpc_transport_t *xprt = NULL;

+        rpc_transport_t *xp_next = NULL;

+        int32_t detach = (long)cookie;

+        gf_boolean_t xprt_found = _gf_false;

         GF_VALIDATE_OR_GOTO ("server", this, out);

         GF_VALIDATE_OR_GOTO ("server", frame, out);

         fd = frame->local;

         client = frame->root->client;

+        serv_xl = frame->this;

+        conf = serv_xl->private;

         fd_unref (fd);

         frame->local = NULL;

+        if (client)

+                victim = client->bound_xl;

+

+        if (victim) {

+                fd_cnt = GF_ATOMIC_DEC(victim->fd_cnt);

+                if (!fd_cnt && conf && detach) {

+                        pthread_mutex_lock(&conf->mutex);

+                        {

+                        list_for_each_entry_safe(xprt, xp_next, &conf->xprt_list, list)

+                        {

+                                if (!xprt->xl_private)

+                                        continue;

+                                if (xprt->xl_private == client) {

+                                        xprt_found = _gf_true;

+                                        break;

+                                }

+                        }

+                        }

+                        pthread_mutex_unlock(&conf->mutex);

+                        if (xprt_found) {

+                                rpc_transport_unref(xprt);

+                        }

+                }

+        }

+

         gf_client_unref (client);

         STACK_DESTROY (frame->root);

@@ -262,7 +297,7 @@ out:

 static int

-do_fd_cleanup (xlator_t *this, client_t* client, fdentry_t *fdentries, int fd_count)

+do_fd_cleanup (xlator_t *this, client_t *client, fdentry_t *fdentries, int fd_count, int32_t detach)

 {

         fd_t               *fd = NULL;

         int                 i = 0, ret = -1;

@@ -307,9 +342,10 @@ do_fd_cleanup (xlator_t *this, client_t* client, fdentry_t *fdentries, int fd_co

                         memset (&tmp_frame->root->lk_owner, 0,

                                 sizeof (gf_lkowner_t));

-                        STACK_WIND (tmp_frame,

-                                    server_connection_cleanup_flush_cbk,

-                                    bound_xl, bound_xl->fops->flush, fd, NULL);

+                        STACK_WIND_COOKIE (tmp_frame,

+                                           server_connection_cleanup_flush_cbk,

+                                           (void *)(long)detach, bound_xl,

+                                           bound_xl->fops->flush, fd, NULL);

                 }

         }

@@ -323,13 +359,18 @@ out:

 int

 server_connection_cleanup (xlator_t *this, client_t *client,

-                           int32_t flags)

+                           int32_t flags, gf_boolean_t *fd_exist)

 {

         server_ctx_t        *serv_ctx  = NULL;

         fdentry_t           *fdentries = NULL;

         uint32_t             fd_count  = 0;

         int                  cd_ret    = 0;

         int                  ret       = 0;

+        xlator_t            *bound_xl = NULL;

+        int                  i = 0;

+        fd_t                *fd = NULL;

+        uint64_t             fd_cnt = 0;

+        int32_t              detach = 0;

         GF_VALIDATE_OR_GOTO (this->name, this, out);

         GF_VALIDATE_OR_GOTO (this->name, client, out);

@@ -360,13 +401,35 @@ server_connection_cleanup (xlator_t *this, client_t *client,

         }

         if (fdentries != NULL) {

+                /* Loop to configure fd_count on victim brick */

+                bound_xl = client->bound_xl;

+                if (bound_xl) {

+                        for (i = 0; i < fd_count; i++) {

+                                fd = fdentries[i].fd;

+                                if (!fd)

+                                        continue;

+                                fd_cnt++;

+                        }

+                        if (fd_cnt) {

+                                if (fd_exist)

+                                        (*fd_exist) = _gf_true;

+                                GF_ATOMIC_ADD(bound_xl->fd_cnt, fd_cnt);

+                        }

+                }

+

+                /* If fd_exist is not NULL it means function is invoke

+                  by server_rpc_notify at the time of getting DISCONNECT

+                  notification

+                */

+                if (fd_exist)

+                        detach = 1;

                 gf_msg_debug (this->name, 0, "Performing cleanup on %d "

                               "fdentries", fd_count);

-                ret = do_fd_cleanup (this, client, fdentries, fd_count);

-        }

-        else

+                ret = do_fd_cleanup (this, client, fdentries, fd_count, detach);

+        } else {

                 gf_msg (this->name, GF_LOG_INFO, 0, PS_MSG_FDENTRY_NULL,

                         "no fdentries to clean");

+        }

         if (cd_ret || ret)

                 ret = -1;

diff --git a/xlators/protocol/server/src/server-helpers.h b/xlators/protocol/server/src/server-helpers.h

index 1f47bc9..84a0cf4 100644

--- a/xlators/protocol/server/src/server-helpers.h

+++ b/xlators/protocol/server/src/server-helpers.h

@@ -42,7 +42,7 @@ get_frame_from_request (rpcsvc_request_t *req);

 int

 server_connection_cleanup (xlator_t *this, struct _client *client,

-                           int32_t flags);

+                           int32_t flags, gf_boolean_t *fd_exist);

 gf_boolean_t

 server_cancel_grace_timer (xlator_t *this, struct _client *client);

diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c

index 69ad184..8ac0bd1 100644

--- a/xlators/protocol/server/src/server.c

+++ b/xlators/protocol/server/src/server.c

@@ -79,7 +79,7 @@ grace_time_handler (void *data)

                 if (detached) /* reconnection did not happen :-( */

                         server_connection_cleanup (this, client,

-                                                   INTERNAL_LOCKS | POSIX_LOCKS);

+                                                   INTERNAL_LOCKS | POSIX_LOCKS, NULL);

                 gf_client_unref (client);

         }

 out:

@@ -195,7 +195,7 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,

                                   "Reply submission failed");

                 if (frame && client && !lk_heal) {

                         server_connection_cleanup (frame->this, client,

-                                                  INTERNAL_LOCKS | POSIX_LOCKS);

+                                                  INTERNAL_LOCKS | POSIX_LOCKS, NULL);

                 } else {

                         gf_msg_callingfn ("", GF_LOG_ERROR, 0,

                                           PS_MSG_REPLY_SUBMIT_FAILED,

@@ -466,6 +466,33 @@ out:

         return error;

 }

+void

+server_call_xlator_mem_cleanup(xlator_t *this, char *victim_name)

+{

+        pthread_t th_id = { 0, };

+        int th_ret = -1;

+        server_cleanup_xprt_arg_t *arg = NULL;

+

+        if (!victim_name)

+                return;

+

+        gf_log(this->name, GF_LOG_INFO, "Create graph janitor thread for brick %s",

+               victim_name);

+

+        arg = calloc(1, sizeof(*arg));

+        arg->this = this;

+        arg->victim_name = gf_strdup(victim_name);

+        th_ret = gf_thread_create_detached(&th_id, server_graph_janitor_threads,

+                                           arg, "graphjanitor");

+        if (th_ret) {

+                gf_log(this->name, GF_LOG_ERROR,

+                       "graph janitor Thread"

+                       " creation is failed for brick %s",

+                       victim_name);

+                GF_FREE(arg->victim_name);

+                free(arg);

+        }

+}

 int

 server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,

@@ -480,14 +507,9 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,

         struct timespec     grace_ts    = {0, };

         char                *auth_path  = NULL;

         int                  ret        = -1;

-        gf_boolean_t         victim_found = _gf_false;

         char                *xlator_name  = NULL;

-        glusterfs_ctx_t     *ctx          = NULL;

-        xlator_t            *top          = NULL;

-        xlator_list_t      **trav_p       = NULL;

-        xlator_t            *travxl       = NULL;

         uint64_t             xprtrefcount = 0;

-        struct _child_status *tmp         = NULL;

+        gf_boolean_t         fd_exist = _gf_false;

         if (!xl || !data) {

@@ -500,7 +522,6 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,

         this = xl;

         trans = data;

         conf = this->private;

-        ctx = glusterfsd_ctx;

         switch (event) {

         case RPCSVC_EVENT_ACCEPT:

@@ -538,7 +559,8 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,

                  */

                 pthread_mutex_lock (&conf->mutex);

                 client = trans->xl_private;

-                list_del_init (&trans->list);

+                if (!client)

+                        list_del_init (&trans->list);

                 pthread_mutex_unlock (&conf->mutex);

                 if (!client)

@@ -563,7 +585,7 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,

                         gf_client_put (client, &detached);

                         if (detached) {

                                 server_connection_cleanup (this, client,

-                                                           INTERNAL_LOCKS | POSIX_LOCKS);

+                                                           INTERNAL_LOCKS | POSIX_LOCKS, &fd_exist);

                                 gf_event (EVENT_CLIENT_DISCONNECT,

                                           "client_uid=%s;"

@@ -638,56 +660,38 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,

 unref_transport:

                 /* rpc_transport_unref() causes a RPCSVC_EVENT_TRANSPORT_DESTROY

                  * to be called in blocking manner

-                 * So no code should ideally be after this unref

+                 * So no code should ideally be after this unref, Call rpc_transport_unref

+                 * only while no client exist or client is not detached or no fd associated

+                   with client

                  */

-                rpc_transport_unref (trans);

+                if (!client || !detached || !fd_exist)

+                        rpc_transport_unref (trans);

                 break;

         case RPCSVC_EVENT_TRANSPORT_DESTROY:

+                pthread_mutex_lock(&conf->mutex);

                 client = trans->xl_private;

+                list_del_init(&trans->list);

+                pthread_mutex_unlock(&conf->mutex);

                 if (!client)

                         break;

-                pthread_mutex_lock (&conf->mutex);

-                list_for_each_entry (tmp, &conf->child_status->status_list,

-                                     status_list) {

-                        if (tmp->name && client->bound_xl &&

-                            client->bound_xl->cleanup_starting &&

-                            !strcmp (tmp->name, client->bound_xl->name)) {

-                                xprtrefcount = GF_ATOMIC_GET (tmp->xprtrefcnt);

-                                if (xprtrefcount > 0) {

-                                        xprtrefcount = GF_ATOMIC_DEC (tmp->xprtrefcnt);

-                                        if (xprtrefcount == 0)

-                                                xlator_name = gf_strdup(client->bound_xl->name);

-                                }

-                                break;

+

+                if (client->bound_xl && client->bound_xl->cleanup_starting) {

+                        xprtrefcount = GF_ATOMIC_GET (client->bound_xl->xprtrefcnt);

+                        if (xprtrefcount > 0) {

+                                xprtrefcount = GF_ATOMIC_DEC (client->bound_xl->xprtrefcnt);

+                                if (xprtrefcount == 0)

+                                        xlator_name = gf_strdup(client->bound_xl->name);

                         }

                 }

-                pthread_mutex_unlock (&conf->mutex);

                 /* unref only for if (!client->lk_heal) */

                 if (!conf->lk_heal)

                         gf_client_unref (client);

                 if (xlator_name) {

-                        if (this->ctx->active) {

-                                top = this->ctx->active->first;

-                                LOCK (&ctx->volfile_lock);

-                                for (trav_p = &top->children; *trav_p;

-                                                   trav_p = &(*trav_p)->next) {

-                                        travxl = (*trav_p)->xlator;

-                                        if (!travxl->call_cleanup &&

-                                            strcmp (travxl->name, xlator_name) == 0) {

-                                                victim_found = _gf_true;

-                                                break;

-                                        }

-                                }

-                                UNLOCK (&ctx->volfile_lock);

-                                if (victim_found) {

-                                        xlator_mem_cleanup (travxl);

-                                        glusterfs_autoscale_threads (ctx, -1);

-                                }

-                        }

+                        server_call_xlator_mem_cleanup (this, xlator_name);

                         GF_FREE (xlator_name);

                 }

@@ -701,6 +705,67 @@ out:

         return 0;

 }

+void *

+server_graph_janitor_threads(void *data)