From c8e58e3a577e70a64df77fe885847285f682d9fb Mon Sep 17 00:00:00 2001

From: Niels de Vos <ndevos@redhat.com>

Date: Tue, 29 Aug 2017 00:16:22 +0200

Subject: [PATCH 492/493] mem-pool: track glusterfs_ctx_t in struct mem_pool

In order to generate statedumps per glusterfs_ctx_t, it is needed to

place all the memory pools in a structure that the context can reach.

The 'struct mem_pool' has been extended with a 'list_head owner' that is

linked with the glusterfs_ctx_t->mempool_list.

All callers of mem_pool_new() have been updated to pass the current

glusterfs_ctx_t along. This context is needed to add the new memory pool

to the list and for grabbing the ctx->lock while updating the

glusterfs_ctx_t->mempool_list.

> Updates: #307

> Change-Id: Ia9384424d8d1630ef3efc9d5d523bf739c356c6e

> Signed-off-by: Niels de Vos <ndevos@redhat.com>

> Reviewed-on: https://review.gluster.org/18075

> Smoke: Gluster Build System <jenkins@build.gluster.org>

> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

> Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>

Change-Id: Id3b193f366f7c46f91b77bced8729a4eb538837b

BUG: 1648893

Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/158937

Tested-by: RHGS Build Bot <nigelb@redhat.com>

Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>

---

 libglusterfs/src/ctx.c                      |  2 --

 libglusterfs/src/glusterfs.h                |  2 --

 libglusterfs/src/mem-pool.c                 | 18 +++++++++++++++++-

 libglusterfs/src/mem-pool.h                 | 16 +++++++++++++---

 libglusterfs/src/rbthash.c                  |  5 +++--

 libglusterfs/src/rbthash.h                  |  2 +-

 libglusterfs/src/statedump.c                | 16 +++++++++++++++-

 xlators/cluster/ec/src/ec-method.c          |  2 +-

 xlators/nfs/server/src/nfs3.c               |  3 ++-

 xlators/performance/io-cache/src/io-cache.c |  2 +-

 10 files changed, 53 insertions(+), 15 deletions(-)

diff --git a/libglusterfs/src/ctx.c b/libglusterfs/src/ctx.c

index 94c56ac..90480d0 100644

--- a/libglusterfs/src/ctx.c

+++ b/libglusterfs/src/ctx.c

@@ -31,9 +31,7 @@ glusterfs_ctx_new ()

         ctx->mem_acct_enable = gf_global_mem_acct_enable_get();

         INIT_LIST_HEAD (&ctx->graphs);

-#if defined(OLD_MEM_POOLS)

         INIT_LIST_HEAD (&ctx->mempool_list);

-#endif

         INIT_LIST_HEAD (&ctx->volfile_list);

 	ctx->daemon_pipe[0] = -1;

diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h

index c12e94e..157437c 100644

--- a/libglusterfs/src/glusterfs.h

+++ b/libglusterfs/src/glusterfs.h

@@ -526,11 +526,9 @@ struct _glusterfs_ctx {

         int                process_mode; /*mode in which process is runninng*/

         struct syncenv    *env;          /* The env pointer to the synctasks */

-#if defined(OLD_MEM_POOLS)

         struct list_head   mempool_list; /* used to keep a global list of

                                             mempools, used to log details of

                                             mempool in statedump */

-#endif

         char              *statedump_path;

         struct mem_pool   *dict_pool;

diff --git a/libglusterfs/src/mem-pool.c b/libglusterfs/src/mem-pool.c

index a8a9347..999a83f 100644

--- a/libglusterfs/src/mem-pool.c

+++ b/libglusterfs/src/mem-pool.c

@@ -676,7 +676,7 @@ void mem_pools_fini (void) {}

 #endif

 struct mem_pool *

-mem_pool_new_fn (unsigned long sizeof_type,

+mem_pool_new_fn (glusterfs_ctx_t *ctx, unsigned long sizeof_type,

                  unsigned long count, char *name)

 {

         unsigned int            i;

@@ -706,10 +706,18 @@ mem_pool_new_fn (unsigned long sizeof_type,

         if (!new)

                 return NULL;

+        new->ctx = ctx;

         new->sizeof_type = sizeof_type;

         new->count = count;

         new->name = name;

         new->pool = pool;

+        INIT_LIST_HEAD (&new->owner);

+

+        LOCK (&ctx->lock);

+        {

+                list_add (&new->owner, &ctx->mempool_list);

+        }

+        UNLOCK (&ctx->lock);

         return new;

 }

@@ -905,6 +913,14 @@ mem_put (void *ptr)

 void

 mem_pool_destroy (struct mem_pool *pool)

 {

+        /* remove this pool from the owner (glusterfs_ctx_t) */

+        LOCK (&pool->ctx->lock);

+        {

+                list_del (&pool->owner);

+        }

+        UNLOCK (&pool->ctx->lock);

+

+        /* free this pool, but keep the mem_pool_shared */

         GF_FREE (pool);

         /*

diff --git a/libglusterfs/src/mem-pool.h b/libglusterfs/src/mem-pool.h

index 057d957..0ebb63b 100644

--- a/libglusterfs/src/mem-pool.h

+++ b/libglusterfs/src/mem-pool.h

@@ -16,6 +16,7 @@

 #include "atomic.h"

 #include "logging.h"

 #include "mem-types.h"

+#include "glusterfs.h" /* for glusterfs_ctx_t */

 #include <stdlib.h>

 #include <inttypes.h>

 #include <string.h>

@@ -207,11 +208,15 @@ out:

 /* kind of 'header' for the actual mem_pool_shared structure, this might make

  * it possible to dump some more details in a statedump */

 struct mem_pool {

+        /* object size, without pooled_obj_hdr_t */

         unsigned long           sizeof_type;

         unsigned long           count;

         char                    *name;

-        struct mem_pool_shared  *pool;

+        struct list_head        owner;  /* glusterfs_ctx_t->mempool_list */

+        glusterfs_ctx_t         *ctx;   /* take ctx->lock when updating owner */

+

+        struct mem_pool_shared  *pool;  /* the initial pool that was returned */

 };

 typedef struct pooled_obj_hdr {

@@ -276,9 +281,14 @@ void mem_pools_init_late (void);   /* start the pool_sweeper thread */

 void mem_pools_fini (void);        /* cleanup memory pools */

 struct mem_pool *

-mem_pool_new_fn (unsigned long sizeof_type, unsigned long count, char *name);

+mem_pool_new_fn (glusterfs_ctx_t *ctx, unsigned long sizeof_type, unsigned long

+                 count, char *name);

+

+#define mem_pool_new(type, count)  (mem_pool_new_fn (THIS->ctx, \

+                                    sizeof(type), count, #type))

-#define mem_pool_new(type,count) mem_pool_new_fn (sizeof(type), count, #type)

+#define mem_pool_new_ctx(ctx, type, count)  (mem_pool_new_fn (ctx, \

+                                             sizeof(type), count, #type))

 void mem_put (void *ptr);

 void *mem_get (struct mem_pool *pool);

diff --git a/libglusterfs/src/rbthash.c b/libglusterfs/src/rbthash.c

index 52d8a15..06fc7ee 100644

--- a/libglusterfs/src/rbthash.c

+++ b/libglusterfs/src/rbthash.c

@@ -83,7 +83,7 @@ err:

  */

 rbthash_table_t *

-rbthash_table_init (int buckets, rbt_hasher_t hfunc,

+rbthash_table_init (glusterfs_ctx_t *ctx, int buckets, rbt_hasher_t hfunc,

                     rbt_data_destroyer_t dfunc,

                     unsigned long expected_entries,

                     struct mem_pool *entrypool)

@@ -123,7 +123,8 @@ rbthash_table_init (int buckets, rbt_hasher_t hfunc,

         if (expected_entries) {

                 newtab->entrypool =

-                        mem_pool_new (rbthash_entry_t, expected_entries);

+                        mem_pool_new_ctx (ctx, rbthash_entry_t,

+                                          expected_entries);

                 if (!newtab->entrypool) {

                         goto free_buckets;

                 }

diff --git a/libglusterfs/src/rbthash.h b/libglusterfs/src/rbthash.h

index b093ce9..949b88a 100644

--- a/libglusterfs/src/rbthash.h

+++ b/libglusterfs/src/rbthash.h

@@ -52,7 +52,7 @@ typedef struct rbthash_table {

 } rbthash_table_t;

 extern rbthash_table_t *

-rbthash_table_init (int buckets, rbt_hasher_t hfunc,

+rbthash_table_init (glusterfs_ctx_t *ctx, int buckets, rbt_hasher_t hfunc,

                     rbt_data_destroyer_t dfunc, unsigned long expected_entries,

                     struct mem_pool *entrypool);

diff --git a/libglusterfs/src/statedump.c b/libglusterfs/src/statedump.c

index a4635f3..4aad014 100644

--- a/libglusterfs/src/statedump.c

+++ b/libglusterfs/src/statedump.c

@@ -377,11 +377,11 @@ gf_proc_dump_mem_info_to_dict (dict_t *dict)

 void

 gf_proc_dump_mempool_info (glusterfs_ctx_t *ctx)

 {

-#if defined(OLD_MEM_POOLS)

         struct mem_pool *pool = NULL;

         gf_proc_dump_add_section ("mempool");

+#if defined(OLD_MEM_POOLS)

         list_for_each_entry (pool, &ctx->mempool_list, global_list) {

                 gf_proc_dump_write ("-----", "-----");

                 gf_proc_dump_write ("pool-name", "%s", pool->name);

@@ -396,6 +396,20 @@ gf_proc_dump_mempool_info (glusterfs_ctx_t *ctx)

                 gf_proc_dump_write ("cur-stdalloc", "%d", pool->curr_stdalloc);

                 gf_proc_dump_write ("max-stdalloc", "%d", pool->max_stdalloc);

         }

+#else

+        LOCK (&ctx->lock);

+        {

+                list_for_each_entry (pool, &ctx->mempool_list, owner) {

+                        gf_proc_dump_write ("-----", "-----");

+                        gf_proc_dump_write ("pool-name", "%s", pool->name);

+                        gf_proc_dump_write ("sizeof-type", "%lu", pool->sizeof_type);

+                        gf_proc_dump_write ("padded-sizeof", "%d", 1 << pool->pool->power_of_two);

+                        gf_proc_dump_write ("shared-pool", "%p", pool->pool);

+                }

+        }

+        UNLOCK (&ctx->lock);

+

+        /* TODO: details of (struct mem_pool_shared) pool->pool */

 #endif

 }

diff --git a/xlators/cluster/ec/src/ec-method.c b/xlators/cluster/ec/src/ec-method.c

index e0dd8e7..a2dd2bd 100644

--- a/xlators/cluster/ec/src/ec-method.c

+++ b/xlators/cluster/ec/src/ec-method.c

@@ -310,7 +310,7 @@ ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns,

     INIT_LIST_HEAD(&list->lru);

     int32_t err;

-    list->pool = mem_pool_new_fn(sizeof(ec_matrix_t) +

+    list->pool = mem_pool_new_fn(xl->ctx, sizeof(ec_matrix_t) +

                                  sizeof(ec_matrix_row_t) * columns +

                                  sizeof(uint32_t) * columns * columns,

                                  128, "ec_matrix_t");

diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c

index 040d316..b053eb3 100644

--- a/xlators/nfs/server/src/nfs3.c

+++ b/xlators/nfs/server/src/nfs3.c

@@ -5786,7 +5786,8 @@ nfs3_init_state (xlator_t *nfsx)

         localpool = nfs->memfactor * GF_NFS_CONCURRENT_OPS_MULT;

         gf_msg_trace (GF_NFS3, 0, "local pool: %d", localpool);

-        nfs3->localpool = mem_pool_new (nfs3_call_state_t, localpool);

+        nfs3->localpool = mem_pool_new_ctx (nfsx->ctx, nfs3_call_state_t,

+                                            localpool);

         if (!nfs3->localpool) {

                 gf_msg (GF_NFS3, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,

                         "local mempool creation failed");

diff --git a/xlators/performance/io-cache/src/io-cache.c b/xlators/performance/io-cache/src/io-cache.c

index de44ad2..d7b3b37 100644

--- a/xlators/performance/io-cache/src/io-cache.c

+++ b/xlators/performance/io-cache/src/io-cache.c

@@ -1146,7 +1146,7 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,

                 if (!ioc_inode->cache.page_table) {

                         ioc_inode->cache.page_table

                                 = rbthash_table_init

-                                (IOC_PAGE_TABLE_BUCKET_COUNT,

+                                (this->ctx, IOC_PAGE_TABLE_BUCKET_COUNT,

                                  ioc_hashfn, NULL, 0,

                                  table->mem_pool);

-- 

1.8.3.1