From 17a9ce965ef2fec9ee5c8e4b76981bb7cbcf1352 Mon Sep 17 00:00:00 2001

From: mohit84 <moagrawa@redhat.com>

Date: Mon, 9 Nov 2020 17:15:42 +0530

Subject: [PATCH 506/511] posix: Attach a posix_spawn_disk_thread with

 glusterfs_ctx (#1595)

Currently posix xlator spawns posix_disk_space_threads per brick and in

case of brick_mux environment while glusterd attached bricks at maximum

level(250) with a single brick process in that case 250 threads are

spawned for all bricks and brick process memory size also increased.

Solution: Attach a posix_disk_space thread with glusterfs_ctx to

          spawn a thread per process basis instead of spawning a per brick

> Fixes: #1482

> Change-Id: I8dd88f252a950495b71742e2a7588bd5bb019ec7

> Cherry-picked from commit 3f93be77e1acf5baacafa97a320e91e6879d1c0e

> Reviewed on upstream link https://github.com/gluster/glusterfs/issues/1482

> Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

Change-Id: I8dd88f252a950495b71742e2a7588bd5bb019ec7

Bug: 1898776

Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/220366

Tested-by: RHGS Build Bot <nigelb@redhat.com>

Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>

---

 glusterfsd/src/glusterfsd.c                    |   4 +

 libglusterfs/src/glusterfs/glusterfs.h         |   6 ++

 xlators/storage/posix/src/posix-common.c       |  68 +++++++++++--

 xlators/storage/posix/src/posix-handle.h       |   3 +-

 xlators/storage/posix/src/posix-helpers.c      | 131 ++++++++++++++-----------

 xlators/storage/posix/src/posix-inode-fd-ops.c |   3 +-

 xlators/storage/posix/src/posix-mem-types.h    |   1 +

 xlators/storage/posix/src/posix.h              |  12 ++-

 8 files changed, 160 insertions(+), 68 deletions(-)

diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c

index 955bf1d..ac25255 100644

--- a/glusterfsd/src/glusterfsd.c

+++ b/glusterfsd/src/glusterfsd.c

@@ -1840,9 +1840,13 @@ glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)

     INIT_LIST_HEAD(&cmd_args->xlator_options);

     INIT_LIST_HEAD(&cmd_args->volfile_servers);

     ctx->pxl_count = 0;

+    ctx->diskxl_count = 0;

     pthread_mutex_init(&ctx->fd_lock, NULL);

     pthread_cond_init(&ctx->fd_cond, NULL);

     INIT_LIST_HEAD(&ctx->janitor_fds);

+    pthread_mutex_init(&ctx->xl_lock, NULL);

+    pthread_cond_init(&ctx->xl_cond, NULL);

+    INIT_LIST_HEAD(&ctx->diskth_xl);

     lim.rlim_cur = RLIM_INFINITY;

     lim.rlim_max = RLIM_INFINITY;

diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h

index bf6a987..d3400bf 100644

--- a/libglusterfs/src/glusterfs/glusterfs.h

+++ b/libglusterfs/src/glusterfs/glusterfs.h

@@ -740,7 +740,13 @@ struct _glusterfs_ctx {

     pthread_t janitor;

     /* The variable is use to save total posix xlator count */

     uint32_t pxl_count;

+    uint32_t diskxl_count;

+    /* List of posix xlator use by disk thread*/

+    struct list_head diskth_xl;

+    pthread_mutex_t xl_lock;

+    pthread_cond_t xl_cond;

+    pthread_t disk_space_check;

     char volume_id[GF_UUID_BUF_SIZE]; /* Used only in protocol/client */

 };

 typedef struct _glusterfs_ctx glusterfs_ctx_t;

diff --git a/xlators/storage/posix/src/posix-common.c b/xlators/storage/posix/src/posix-common.c

index e5c6e62..2c9030b 100644

--- a/xlators/storage/posix/src/posix-common.c

+++ b/xlators/storage/posix/src/posix-common.c

@@ -138,6 +138,36 @@ posix_inode(xlator_t *this)

     return 0;

 }

+static void

+delete_posix_diskxl(xlator_t *this)

+{

+    struct posix_private *priv = this->private;

+    struct posix_diskxl *pxl = priv->pxl;

+    glusterfs_ctx_t *ctx = this->ctx;

+    uint32_t count = 1;

+

+    if (pxl) {

+        pthread_mutex_lock(&ctx->xl_lock);

+        {

+            pxl->detach_notify = _gf_true;

+            while (pxl->is_use)

+                pthread_cond_wait(&pxl->cond, &ctx->xl_lock);

+            list_del_init(&pxl->list);

+            priv->pxl = NULL;

+            count = --ctx->diskxl_count;

+            if (count == 0)

+                pthread_cond_signal(&ctx->xl_cond);

+        }

+        pthread_mutex_unlock(&ctx->xl_lock);

+        pthread_cond_destroy(&pxl->cond);

+        GF_FREE(pxl);

+        if (count == 0) {

+            pthread_join(ctx->disk_space_check, NULL);

+            ctx->disk_space_check = 0;

+        }

+    }

+}

+

 /**

  * notify - when parent sends PARENT_UP, send CHILD_UP event from here

  */

@@ -194,6 +224,8 @@ posix_notify(xlator_t *this, int32_t event, void *data, ...)

             }

             pthread_mutex_unlock(&ctx->fd_lock);

+            delete_posix_diskxl(this);

+

             gf_log(this->name, GF_LOG_INFO, "Sending CHILD_DOWN for brick %s",

                    victim->name);

             default_notify(this->parents->xlator, GF_EVENT_CHILD_DOWN, data);

@@ -318,6 +350,7 @@ posix_reconfigure(xlator_t *this, dict_t *options)

     int32_t force_directory_mode = -1;

     int32_t create_mask = -1;

     int32_t create_directory_mask = -1;

+    double old_disk_reserve = 0.0;

     priv = this->private;

@@ -383,6 +416,7 @@ posix_reconfigure(xlator_t *this, dict_t *options)

                " fallback to <hostname>:<export>");

     }

+    old_disk_reserve = priv->disk_reserve;

     GF_OPTION_RECONF("reserve", priv->disk_reserve, options, percent_or_size,

                      out);

     /* option can be any one of percent or bytes */

@@ -390,11 +424,19 @@ posix_reconfigure(xlator_t *this, dict_t *options)

     if (priv->disk_reserve < 100.0)

         priv->disk_unit = 'p';

-    if (priv->disk_reserve) {

+    /* Delete a pxl object from a list of disk_reserve while something

+       is changed for reserve option during graph reconfigure

+    */

+    if (old_disk_reserve != priv->disk_reserve) {

+        delete_posix_diskxl(this);

+        old_disk_reserve = 0;

+    }

+

+    if (!old_disk_reserve && priv->disk_reserve) {

         ret = posix_spawn_disk_space_check_thread(this);

         if (ret) {

             gf_msg(this->name, GF_LOG_INFO, 0, P_MSG_DISK_SPACE_CHECK_FAILED,

-                   "Getting disk space check from thread failed");

+                   "Getting disk space check from thread failed ");

             goto out;

         }

     }

@@ -1008,13 +1050,13 @@ posix_init(xlator_t *this)

                " fallback to <hostname>:<export>");

     }

-    _private->disk_space_check_active = _gf_false;

     _private->disk_space_full = 0;

     GF_OPTION_INIT("reserve", _private->disk_reserve, percent_or_size, out);

     /* option can be any one of percent or bytes */

     _private->disk_unit = 0;

+    pthread_cond_init(&_private->fd_cond, NULL);

     if (_private->disk_reserve < 100.0)

         _private->disk_unit = 'p';

@@ -1162,12 +1204,6 @@ posix_fini(xlator_t *this)

         priv->health_check = 0;

     }

-    if (priv->disk_space_check) {

-        priv->disk_space_check_active = _gf_false;

-        (void)gf_thread_cleanup_xint(priv->disk_space_check);

-        priv->disk_space_check = 0;

-    }

-

     if (priv->janitor) {

         /*TODO: Make sure the synctask is also complete */

         ret = gf_tw_del_timer(this->ctx->tw->timer_wheel, priv->janitor);

@@ -1192,10 +1228,24 @@ posix_fini(xlator_t *this)

         pthread_join(ctx->janitor, NULL);

     }

+    pthread_mutex_lock(&ctx->xl_lock);

+    {

+        count = --ctx->diskxl_count;

+        if (count == 0)

+            pthread_cond_signal(&ctx->xl_cond);

+    }

+    pthread_mutex_unlock(&ctx->xl_lock);

+

+    if (count == 0) {

+        pthread_join(ctx->disk_space_check, NULL);

+        ctx->disk_space_check = 0;

+    }

+

     if (priv->fsyncer) {

         (void)gf_thread_cleanup_xint(priv->fsyncer);

         priv->fsyncer = 0;

     }

+

     /*unlock brick dir*/

     if (priv->mount_lock)

         (void)sys_closedir(priv->mount_lock);

diff --git a/xlators/storage/posix/src/posix-handle.h b/xlators/storage/posix/src/posix-handle.h

index c4d7cb1..8e4c719 100644

--- a/xlators/storage/posix/src/posix-handle.h

+++ b/xlators/storage/posix/src/posix-handle.h

@@ -206,5 +206,6 @@ int

 posix_check_internal_writes(xlator_t *this, fd_t *fd, int sysfd, dict_t *xdata);

 void

-posix_disk_space_check(xlator_t *this);

+posix_disk_space_check(struct posix_private* priv);

+

 #endif /* !_POSIX_HANDLE_H */

diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c

index ceac52a..110d383 100644

--- a/xlators/storage/posix/src/posix-helpers.c

+++ b/xlators/storage/posix/src/posix-helpers.c

@@ -2284,9 +2284,8 @@ unlock:

 }

 void

-posix_disk_space_check(xlator_t *this)

+posix_disk_space_check(struct posix_private *priv)

 {

-    struct posix_private *priv = NULL;

     char *subvol_path = NULL;

     int op_ret = 0;

     double size = 0;

@@ -2295,16 +2294,14 @@ posix_disk_space_check(xlator_t *this)

     double totsz = 0;

     double freesz = 0;

-    GF_VALIDATE_OR_GOTO(this->name, this, out);

-    priv = this->private;

-    GF_VALIDATE_OR_GOTO(this->name, priv, out);

+    GF_VALIDATE_OR_GOTO("posix-helpers", priv, out);

     subvol_path = priv->base_path;

     op_ret = sys_statvfs(subvol_path, &buf;;

     if (op_ret == -1) {

-        gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_STATVFS_FAILED,

+        gf_msg("posix-disk", GF_LOG_ERROR, errno, P_MSG_STATVFS_FAILED,

                "statvfs failed on %s", subvol_path);

         goto out;

     }

@@ -2328,78 +2325,102 @@ out:

 }

 static void *

-posix_disk_space_check_thread_proc(void *data)

+posix_ctx_disk_thread_proc(void *data)

 {

-    xlator_t *this = NULL;

     struct posix_private *priv = NULL;

+    glusterfs_ctx_t *ctx = NULL;

     uint32_t interval = 0;

-    int ret = -1;

-

-    this = data;

-    priv = this->private;

+    struct posix_diskxl *pthis = NULL;

+    xlator_t *this = NULL;

+    struct timespec sleep_till = {

+        0,

+    };

+    ctx = data;

     interval = 5;

-    gf_msg_debug(this->name, 0,

-                 "disk-space thread started, "

+

+    gf_msg_debug("glusterfs_ctx", 0,

+                 "Ctx disk-space thread started, "

                  "interval = %d seconds",

                  interval);

-    while (1) {

-        /* aborting sleep() is a request to exit this thread, sleep()

-         * will normally not return when cancelled */

-        ret = sleep(interval);

-        if (ret > 0)

-            break;

-        /* prevent thread errors while doing the health-check(s) */

-        pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);

-

-        /* Do the disk-check.*/

-        posix_disk_space_check(this);

-        if (!priv->disk_space_check_active)

-            goto out;

-        pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);

-    }

-out:

-    gf_msg_debug(this->name, 0, "disk space check thread exiting");

-    LOCK(&priv->lock);

+    pthread_mutex_lock(&ctx->xl_lock);

     {

-        priv->disk_space_check_active = _gf_false;

+        while (ctx->diskxl_count > 0) {

+            list_for_each_entry(pthis, &ctx->diskth_xl, list)

+            {

+                pthis->is_use = _gf_true;

+                pthread_mutex_unlock(&ctx->xl_lock);

+

+                THIS = this = pthis->xl;

+                priv = this->private;

+

+                posix_disk_space_check(priv);

+

+                pthread_mutex_lock(&ctx->xl_lock);

+                pthis->is_use = _gf_false;

+                /* Send a signal to posix_notify function */

+                if (pthis->detach_notify)

+                    pthread_cond_signal(&pthis->cond);

+            }

+

+            timespec_now_realtime(&sleep_till);

+            sleep_till.tv_sec += 5;

+            (void)pthread_cond_timedwait(&ctx->xl_cond, &ctx->xl_lock,

+                                         &sleep_till);

+        }

     }

-    UNLOCK(&priv->lock);

+    pthread_mutex_unlock(&ctx->xl_lock);

     return NULL;

 }

 int

-posix_spawn_disk_space_check_thread(xlator_t *xl)

+posix_spawn_disk_space_check_thread(xlator_t *this)

 {

-    struct posix_private *priv = NULL;

-    int ret = -1;

+    int ret = 0;

+    glusterfs_ctx_t *ctx = this->ctx;

+    struct posix_diskxl *pxl = NULL;

+    struct posix_private *priv = this->private;

-    priv = xl->private;

+    pxl = GF_CALLOC(1, sizeof(struct posix_diskxl), gf_posix_mt_diskxl_t);

+    if (!pxl) {

+        ret = -ENOMEM;

+        gf_log(this->name, GF_LOG_ERROR,

+               "Calloc is failed to allocate "

+               "memory for diskxl object");

+        goto out;

+    }

+    pthread_cond_init(&pxl->cond, NULL);

-    LOCK(&priv->lock);

+    pthread_mutex_lock(&ctx->xl_lock);

     {

-        /* cancel the running thread  */

-        if (priv->disk_space_check_active == _gf_true) {

-            pthread_cancel(priv->disk_space_check);

-            priv->disk_space_check_active = _gf_false;

-        }

+        if (ctx->diskxl_count++ == 0) {

+            ret = gf_thread_create(&ctx->disk_space_check, NULL,

+                                   posix_ctx_disk_thread_proc, ctx,

+                                   "posixctxres");

-        ret = gf_thread_create(&priv->disk_space_check, NULL,

-                               posix_disk_space_check_thread_proc, xl,

-                               "posix_reserve");

-        if (ret) {

-            priv->disk_space_check_active = _gf_false;

-            gf_msg(xl->name, GF_LOG_ERROR, errno, P_MSG_DISK_SPACE_CHECK_FAILED,

-                   "unable to setup disk space check thread");

-            goto unlock;

+            if (ret) {

+                gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED,

+                       "spawning disk space check thread failed");

+                ctx->diskxl_count--;

+                pthread_mutex_unlock(&ctx->xl_lock);

+                goto out;

+            }

         }

+        pxl->xl = this;

+        priv->pxl = (void *)pxl;

+        list_add_tail(&pxl->list, &ctx->diskth_xl);

+    }

+    pthread_mutex_unlock(&ctx->xl_lock);

-        priv->disk_space_check_active = _gf_true;

+out:

+    if (ret) {

+        if (pxl) {

+            pthread_cond_destroy(&pxl->cond);

+            GF_FREE(pxl);

+        }

     }

-unlock:

-    UNLOCK(&priv->lock);

     return ret;

 }

diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c

index 1d37aed..761e018 100644

--- a/xlators/storage/posix/src/posix-inode-fd-ops.c

+++ b/xlators/storage/posix/src/posix-inode-fd-ops.c

@@ -37,6 +37,7 @@

 #include <fcntl.h>

 #endif /* HAVE_LINKAT */

+#include "posix-handle.h"

 #include <glusterfs/glusterfs.h>

 #include <glusterfs/checksum.h>

 #include <glusterfs/dict.h>

@@ -713,7 +714,7 @@ posix_do_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,

        option behaviour

     */

     if (priv->disk_reserve)

-        posix_disk_space_check(this);

+        posix_disk_space_check(priv);

     DISK_SPACE_CHECK_AND_GOTO(frame, priv, xdata, ret, ret, unlock);

diff --git a/xlators/storage/posix/src/posix-mem-types.h b/xlators/storage/posix/src/posix-mem-types.h

index 2253f38..bb4c56d 100644

--- a/xlators/storage/posix/src/posix-mem-types.h

+++ b/xlators/storage/posix/src/posix-mem-types.h

@@ -20,6 +20,7 @@ enum gf_posix_mem_types_ {

     gf_posix_mt_paiocb,

     gf_posix_mt_inode_ctx_t,

     gf_posix_mt_mdata_attr,

+    gf_posix_mt_diskxl_t,

     gf_posix_mt_end

 };

 #endif

diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h

index 07f367b..4be979c 100644

--- a/xlators/storage/posix/src/posix.h

+++ b/xlators/storage/posix/src/posix.h

@@ -36,7 +36,6 @@

 #include <glusterfs/compat.h>

 #include <glusterfs/timer.h>

 #include "posix-mem-types.h"

-#include "posix-handle.h"

 #include <glusterfs/call-stub.h>

 #ifdef HAVE_LIBAIO

@@ -138,6 +137,14 @@ struct posix_fd {

     char _pad[4]; /* manual padding */

 };

+struct posix_diskxl {

+    pthread_cond_t cond;

+    struct list_head list;

+    xlator_t *xl;

+    gf_boolean_t detach_notify;

+    gf_boolean_t is_use;

+};

+

 struct posix_private {

     char *base_path;

     int32_t base_path_length;

@@ -207,6 +214,7 @@ struct posix_private {

     pthread_mutex_t janitor_mutex;

     pthread_cond_t janitor_cond;

     pthread_cond_t fd_cond;

+    pthread_cond_t disk_cond;

     int fsync_queue_count;

     enum {

@@ -233,7 +241,6 @@ struct posix_private {

     char disk_unit;

     uint32_t disk_space_full;

     pthread_t disk_space_check;

-    gf_boolean_t disk_space_check_active;

 #ifdef GF_DARWIN_HOST_OS

     enum {

@@ -263,6 +270,7 @@ struct posix_private {

     gf_boolean_t ctime;

     gf_boolean_t janitor_task_stop;

     uint32_t rel_fdcount;

+    void *pxl;

 };

 typedef struct {

-- 

1.8.3.1