From a18f03cbf2b5652f8617cb4dd236bb4ca9838d96 Mon Sep 17 00:00:00 2001
From: Mohit Agrawal <moagrawa@redhat.com>
Date: Tue, 6 Oct 2020 16:54:15 +0530
Subject: [PATCH 509/511] core: configure optimum inode table hash_size for shd
In brick_mux environment a shd process consume high memory.
After print the statedump i have found it allocates 1M per afr xlator
for all bricks.In case of configure 4k volumes it consumes almost total
6G RSS size in which 4G consumes by inode_tables
[cluster/replicate.test1-replicate-0 - usage-type gf_common_mt_list_head memusage]
size=1273488
num_allocs=2
max_size=1273488
max_num_allocs=2
total_allocs=2
inode_new_table function allocates memory(1M) for a list of inode and dentry hash.
For shd lru_limit size is 1 so we don't need to create a big hash table so to reduce
RSS size for shd process pass optimum bucket count at the time of creating inode_table.
> Change-Id: I039716d42321a232fdee1ee8fd50295e638715bb
> Fixes: #1538
> Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>
> (Cherry pick from commit ca6bbc486e76fdb9a8e07119bb10d7fa45b2e93b)
> (Reviewed on upstream link https://github.com/gluster/glusterfs/issues/1538)
Change-Id: I039716d42321a232fdee1ee8fd50295e638715bb
BUG: 1898777
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/221191
Tested-by: RHGS Build Bot <nigelb@redhat.com>
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
---
api/src/glfs-master.c | 2 +-
libglusterfs/src/glusterfs/inode.h | 17 +++++----
libglusterfs/src/inode.c | 53 +++++++++++++++++---------
xlators/cluster/afr/src/afr.c | 10 ++++-
xlators/cluster/dht/src/dht-rebalance.c | 3 +-
xlators/cluster/ec/src/ec.c | 2 +-
xlators/features/bit-rot/src/bitd/bit-rot.c | 2 +-
xlators/features/quota/src/quotad-helpers.c | 2 +-
xlators/features/trash/src/trash.c | 4 +-
xlators/mount/fuse/src/fuse-bridge.c | 6 +--
xlators/nfs/server/src/nfs.c | 2 +-
xlators/protocol/server/src/server-handshake.c | 3 +-
12 files changed, 66 insertions(+), 40 deletions(-)
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
index b4473b1..9e604d3 100644
--- a/api/src/glfs-master.c
+++ b/api/src/glfs-master.c
@@ -45,7 +45,7 @@ graph_setup(struct glfs *fs, glusterfs_graph_t *graph)
}
if (!new_subvol->itable) {
- itable = inode_table_new(131072, new_subvol);
+ itable = inode_table_new(131072, new_subvol, 0, 0);
if (!itable) {
errno = ENOMEM;
ret = -1;
diff --git a/libglusterfs/src/glusterfs/inode.h b/libglusterfs/src/glusterfs/inode.h
index c875653..62c093d 100644
--- a/libglusterfs/src/glusterfs/inode.h
+++ b/libglusterfs/src/glusterfs/inode.h
@@ -35,11 +35,12 @@ typedef struct _dentry dentry_t;
struct _inode_table {
pthread_mutex_t lock;
- size_t hashsize; /* bucket size of inode hash and dentry hash */
- char *name; /* name of the inode table, just for gf_log() */
- inode_t *root; /* root directory inode, with number 1 */
- xlator_t *xl; /* xlator to be called to do purge */
- uint32_t lru_limit; /* maximum LRU cache size */
+ size_t dentry_hashsize; /* Number of buckets for dentry hash*/
+ size_t inode_hashsize; /* Size of inode hash table */
+ char *name; /* name of the inode table, just for gf_log() */
+ inode_t *root; /* root directory inode, with number 1 */
+ xlator_t *xl; /* xlator to be called to do purge */
+ uint32_t lru_limit; /* maximum LRU cache size */
struct list_head *inode_hash; /* buckets for inode hash table */
struct list_head *name_hash; /* buckets for dentry hash table */
struct list_head active; /* list of inodes currently active (in an fop) */
@@ -116,12 +117,14 @@ struct _inode {
#define GFID_STR_PFX_LEN (sizeof(GFID_STR_PFX) - 1)
inode_table_t *
-inode_table_new(uint32_t lru_limit, xlator_t *xl);
+inode_table_new(uint32_t lru_limit, xlator_t *xl, uint32_t dhash_size,
+ uint32_t inodehash_size);
inode_table_t *
inode_table_with_invalidator(uint32_t lru_limit, xlator_t *xl,
int32_t (*invalidator_fn)(xlator_t *, inode_t *),
- xlator_t *invalidator_xl);
+ xlator_t *invalidator_xl, uint32_t dentry_hashsize,
+ uint32_t inode_hashsize);
void
inode_table_destroy_all(glusterfs_ctx_t *ctx);
diff --git a/libglusterfs/src/inode.c b/libglusterfs/src/inode.c
index 71b2d2a..98f8ea6 100644
--- a/libglusterfs/src/inode.c
+++ b/libglusterfs/src/inode.c
@@ -763,7 +763,7 @@ inode_grep(inode_table_t *table, inode_t *parent, const char *name)
return NULL;
}
- int hash = hash_dentry(parent, name, table->hashsize);
+ int hash = hash_dentry(parent, name, table->dentry_hashsize);
pthread_mutex_lock(&table->lock);
{
@@ -839,7 +839,7 @@ inode_grep_for_gfid(inode_table_t *table, inode_t *parent, const char *name,
return ret;
}
- int hash = hash_dentry(parent, name, table->hashsize);
+ int hash = hash_dentry(parent, name, table->dentry_hashsize);
pthread_mutex_lock(&table->lock);
{
@@ -903,7 +903,7 @@ inode_find(inode_table_t *table, uuid_t gfid)
return NULL;
}
- int hash = hash_gfid(gfid, 65536);
+ int hash = hash_gfid(gfid, table->inode_hashsize);
pthread_mutex_lock(&table->lock);
{
@@ -964,7 +964,7 @@ __inode_link(inode_t *inode, inode_t *parent, const char *name,
return NULL;
}
- int ihash = hash_gfid(iatt->ia_gfid, 65536);
+ int ihash = hash_gfid(iatt->ia_gfid, table->inode_hashsize);
old_inode = __inode_find(table, iatt->ia_gfid, ihash);
@@ -1043,7 +1043,7 @@ inode_link(inode_t *inode, inode_t *parent, const char *name, struct iatt *iatt)
table = inode->table;
if (parent && name) {
- hash = hash_dentry(parent, name, table->hashsize);
+ hash = hash_dentry(parent, name, table->dentry_hashsize);
}
if (name && strchr(name, '/')) {
@@ -1262,7 +1262,7 @@ inode_rename(inode_table_t *table, inode_t *srcdir, const char *srcname,
}
if (dstdir && dstname) {
- hash = hash_dentry(dstdir, dstname, table->hashsize);
+ hash = hash_dentry(dstdir, dstname, table->dentry_hashsize);
}
pthread_mutex_lock(&table->lock);
@@ -1626,7 +1626,8 @@ __inode_table_init_root(inode_table_t *table)
inode_table_t *
inode_table_with_invalidator(uint32_t lru_limit, xlator_t *xl,
int32_t (*invalidator_fn)(xlator_t *, inode_t *),
- xlator_t *invalidator_xl)
+ xlator_t *invalidator_xl, uint32_t dentry_hashsize,
+ uint32_t inode_hashsize)
{
inode_table_t *new = NULL;
uint32_t mem_pool_size = lru_limit;
@@ -1644,7 +1645,19 @@ inode_table_with_invalidator(uint32_t lru_limit, xlator_t *xl,
new->invalidator_fn = invalidator_fn;
new->invalidator_xl = invalidator_xl;
- new->hashsize = 14057; /* TODO: Random Number?? */
+ if (dentry_hashsize == 0) {
+ /* Prime number for uniform distribution */
+ new->dentry_hashsize = 14057;
+ } else {
+ new->dentry_hashsize = dentry_hashsize;
+ }
+
+ if (inode_hashsize == 0) {
+ /* The size of hash table always should be power of 2 */
+ new->inode_hashsize = 65536;
+ } else {
+ new->inode_hashsize = inode_hashsize;
+ }
/* In case FUSE is initing the inode table. */
if (!mem_pool_size || (mem_pool_size > DEFAULT_INODE_MEMPOOL_ENTRIES))
@@ -1658,13 +1671,13 @@ inode_table_with_invalidator(uint32_t lru_limit, xlator_t *xl,
if (!new->dentry_pool)
goto out;
- new->inode_hash = (void *)GF_CALLOC(65536, sizeof(struct list_head),
- gf_common_mt_list_head);
+ new->inode_hash = (void *)GF_CALLOC(
+ new->inode_hashsize, sizeof(struct list_head), gf_common_mt_list_head);
if (!new->inode_hash)
goto out;
- new->name_hash = (void *)GF_CALLOC(new->hashsize, sizeof(struct list_head),
- gf_common_mt_list_head);
+ new->name_hash = (void *)GF_CALLOC(
+ new->dentry_hashsize, sizeof(struct list_head), gf_common_mt_list_head);
if (!new->name_hash)
goto out;
@@ -1675,11 +1688,11 @@ inode_table_with_invalidator(uint32_t lru_limit, xlator_t *xl,
if (!new->fd_mem_pool)
goto out;
- for (i = 0; i < 65536; i++) {
+ for (i = 0; i < new->inode_hashsize; i++) {
INIT_LIST_HEAD(&new->inode_hash[i]);
}
- for (i = 0; i < new->hashsize; i++) {
+ for (i = 0; i < new->dentry_hashsize; i++) {
INIT_LIST_HEAD(&new->name_hash[i]);
}
@@ -1717,10 +1730,12 @@ out:
}
inode_table_t *
-inode_table_new(uint32_t lru_limit, xlator_t *xl)
+inode_table_new(uint32_t lru_limit, xlator_t *xl, uint32_t dentry_hashsize,
+ uint32_t inode_hashsize)
{
/* Only fuse for now requires the inode table with invalidator */
- return inode_table_with_invalidator(lru_limit, xl, NULL, NULL);
+ return inode_table_with_invalidator(lru_limit, xl, NULL, NULL,
+ dentry_hashsize, inode_hashsize);
}
int
@@ -2439,8 +2454,10 @@ inode_table_dump(inode_table_t *itable, char *prefix)
return;
}
- gf_proc_dump_build_key(key, prefix, "hashsize");
- gf_proc_dump_write(key, "%" GF_PRI_SIZET, itable->hashsize);
+ gf_proc_dump_build_key(key, prefix, "dentry_hashsize");
+ gf_proc_dump_write(key, "%" GF_PRI_SIZET, itable->dentry_hashsize);
+ gf_proc_dump_build_key(key, prefix, "inode_hashsize");
+ gf_proc_dump_write(key, "%" GF_PRI_SIZET, itable->inode_hashsize);
gf_proc_dump_build_key(key, prefix, "name");
gf_proc_dump_write(key, "%s", itable->name);
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 8f9e71f..bfa464f 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -594,7 +594,15 @@ init(xlator_t *this)
goto out;
}
- this->itable = inode_table_new(SHD_INODE_LRU_LIMIT, this);
+ if (priv->shd.iamshd) {
+ /* Number of hash bucket should be prime number so declare 131
+ total dentry hash buckets
+ */
+ this->itable = inode_table_new(SHD_INODE_LRU_LIMIT, this, 131, 128);
+ } else {
+ this->itable = inode_table_new(SHD_INODE_LRU_LIMIT, this, 0, 0);
+ }
+
if (!this->itable) {
ret = -ENOMEM;
goto out;
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 16ac16c..072896d 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -1168,7 +1168,6 @@ __dht_rebalance_migrate_data(xlator_t *this, gf_defrag_info_t *defrag,
break;
}
-
offset += ret;
total += ret;
@@ -2467,7 +2466,7 @@ dht_build_root_inode(xlator_t *this, inode_t **inode)
0,
};
- itable = inode_table_new(0, this);
+ itable = inode_table_new(0, this, 0, 0);
if (!itable)
return;
diff --git a/xlators/cluster/ec/src/ec.c b/xlators/cluster/ec/src/ec.c
index 3f31c74..4118c3b 100644
--- a/xlators/cluster/ec/src/ec.c
+++ b/xlators/cluster/ec/src/ec.c
@@ -734,7 +734,7 @@ init(xlator_t *this)
GF_OPTION_INIT("stripe-cache", ec->stripe_cache, uint32, failed);
GF_OPTION_INIT("quorum-count", ec->quorum_count, uint32, failed);
- this->itable = inode_table_new(EC_SHD_INODE_LRU_LIMIT, this);
+ this->itable = inode_table_new(EC_SHD_INODE_LRU_LIMIT, this, 0, 0);
if (!this->itable)
goto failed;
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c
index 424c0d5..4e0e798 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.c
@@ -1658,7 +1658,7 @@ notify(xlator_t *this, int32_t event, void *data, ...)
child->child_up = 1;
child->xl = subvol;
if (!child->table)
- child->table = inode_table_new(4096, subvol);
+ child->table = inode_table_new(4096, subvol, 0, 0);
_br_qchild_event(this, child, br_brick_connect);
pthread_cond_signal(&priv->cond);
diff --git a/xlators/features/quota/src/quotad-helpers.c b/xlators/features/quota/src/quotad-helpers.c
index d9f0351..46ac116 100644
--- a/xlators/features/quota/src/quotad-helpers.c
+++ b/xlators/features/quota/src/quotad-helpers.c
@@ -32,7 +32,7 @@ get_quotad_aggregator_state(xlator_t *this, rpcsvc_request_t *req)
UNLOCK(&priv->lock);
if (active_subvol->itable == NULL)
- active_subvol->itable = inode_table_new(4096, active_subvol);
+ active_subvol->itable = inode_table_new(4096, active_subvol, 0, 0);
state->itable = active_subvol->itable;
diff --git a/xlators/features/trash/src/trash.c b/xlators/features/trash/src/trash.c
index 93f020f..099c887 100644
--- a/xlators/features/trash/src/trash.c
+++ b/xlators/features/trash/src/trash.c
@@ -2261,7 +2261,7 @@ reconfigure(xlator_t *this, dict_t *options)
if (!active_earlier && active_now) {
if (!priv->trash_itable) {
- priv->trash_itable = inode_table_new(0, this);
+ priv->trash_itable = inode_table_new(0, this, 0, 0);
if (!priv->trash_itable) {
ret = -ENOMEM;
gf_log(this->name, GF_LOG_ERROR,
@@ -2533,7 +2533,7 @@ init(xlator_t *this)
}
if (priv->state) {
- priv->trash_itable = inode_table_new(0, this);
+ priv->trash_itable = inode_table_new(0, this, 0, 0);
if (!priv->trash_itable) {
ret = -ENOMEM;
priv->state = _gf_false;
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 1bddac2..919eea3 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -6298,10 +6298,10 @@ fuse_graph_setup(xlator_t *this, glusterfs_graph_t *graph)
}
#if FUSE_KERNEL_MINOR_VERSION >= 11
- itable = inode_table_with_invalidator(priv->lru_limit, graph->top,
- fuse_inode_invalidate_fn, this);
+ itable = inode_table_with_invalidator(
+ priv->lru_limit, graph->top, fuse_inode_invalidate_fn, this, 0, 0);
#else
- itable = inode_table_new(0, graph->top);
+ itable = inode_table_new(0, graph->top, 0, 0);
#endif
if (!itable) {
ret = -1;
diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c
index ebded41..402be30 100644
--- a/xlators/nfs/server/src/nfs.c
+++ b/xlators/nfs/server/src/nfs.c
@@ -564,7 +564,7 @@ nfs_init_subvolume(struct nfs_state *nfs, xlator_t *xl)
return -1;
lrusize = nfs->memfactor * GF_NFS_INODE_LRU_MULT;
- xl->itable = inode_table_new(lrusize, xl);
+ xl->itable = inode_table_new(lrusize, xl, 0, 0);
if (!xl->itable) {
gf_msg(GF_NFS, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
"Failed to allocate inode table");
diff --git a/xlators/protocol/server/src/server-handshake.c b/xlators/protocol/server/src/server-handshake.c
index 1d1177d..eeca73c 100644
--- a/xlators/protocol/server/src/server-handshake.c
+++ b/xlators/protocol/server/src/server-handshake.c
@@ -36,7 +36,6 @@ gf_compare_client_version(rpcsvc_request_t *req, int fop_prognum,
return ret;
}
-
int
server_getspec(rpcsvc_request_t *req)
{
@@ -629,7 +628,7 @@ server_setvolume(rpcsvc_request_t *req)
/* TODO: what is this ? */
client->bound_xl->itable = inode_table_new(conf->inode_lru_limit,
- client->bound_xl);
+ client->bound_xl, 0, 0);
}
}
UNLOCK(&conf->itable_lock);
--
1.8.3.1