From ef167770c456b228aa4227d9376bcf2c283df95e Mon Sep 17 00:00:00 2001
From: Raghavendra G <rgowdapp@redhat.com>
Date: Thu, 24 Nov 2016 14:58:20 +0530
Subject: [PATCH 255/257] performance/readdir-ahead: limit cache size
This patch introduces a new option called "rda-cache-limit", which is
the maximum value the entire readdir-ahead cache can grow into. Since,
readdir-ahead holds a reference to inode through dentries, this patch
also accounts memory stored by various xlators in inode contexts.
>Reviewed-on: http://review.gluster.org/16137
>NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
>Reviewed-by: Poornima G <pgurusid@redhat.com>
>Smoke: Gluster Build System <jenkins@build.gluster.org>
>CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Change-Id: I84cc0ca812f35e0f9041f8cc71effae53a9e7f99
BUG: 1393316
Signed-off-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/93587
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Tested-by: Atin Mukherjee <amukherj@redhat.com>
---
libglusterfs/src/inode.c | 32 ++++++
libglusterfs/src/inode.h | 3 +
libglusterfs/src/xlator.h | 6 ++
xlators/mgmt/glusterd/src/glusterd-volume-set.c | 28 ++++-
.../performance/readdir-ahead/src/readdir-ahead.c | 116 +++++++++++++++------
.../performance/readdir-ahead/src/readdir-ahead.h | 4 +-
6 files changed, 153 insertions(+), 36 deletions(-)
diff --git a/libglusterfs/src/inode.c b/libglusterfs/src/inode.c
index 2aca41b..650a301 100644
--- a/libglusterfs/src/inode.c
+++ b/libglusterfs/src/inode.c
@@ -2507,3 +2507,35 @@ out:
return;
}
+
+size_t
+inode_ctx_size (inode_t *inode)
+{
+ int i = 0;
+ size_t size = 0;
+ xlator_t *xl = NULL, *old_THIS = NULL;
+
+ if (!inode)
+ goto out;
+
+ LOCK (&inode->lock);
+ {
+ for (i = 0; i < inode->table->ctxcount; i++) {
+ if (!inode->_ctx[i].xl_key)
+ continue;
+
+ xl = (xlator_t *)(long)inode->_ctx[i].xl_key;
+ old_THIS = THIS;
+ THIS = xl;
+
+ if (xl->cbks->ictxsize)
+ size += xl->cbks->ictxsize (xl, inode);
+
+ THIS = old_THIS;
+ }
+ }
+ UNLOCK (&inode->lock);
+
+out:
+ return size;
+}
diff --git a/libglusterfs/src/inode.h b/libglusterfs/src/inode.h
index 114aeae..5289b15 100644
--- a/libglusterfs/src/inode.h
+++ b/libglusterfs/src/inode.h
@@ -279,4 +279,7 @@ inode_needs_lookup (inode_t *inode, xlator_t *this);
int
inode_has_dentry (inode_t *inode);
+size_t
+inode_ctx_size (inode_t *inode);
+
#endif /* _INODE_H */
diff --git a/libglusterfs/src/xlator.h b/libglusterfs/src/xlator.h
index 70e6f0a..b11d1a9 100644
--- a/libglusterfs/src/xlator.h
+++ b/libglusterfs/src/xlator.h
@@ -847,6 +847,10 @@ typedef int32_t (*cbk_client_t)(xlator_t *this, client_t *client);
typedef void (*cbk_ictxmerge_t) (xlator_t *this, fd_t *fd,
inode_t *inode, inode_t *linked_inode);
+typedef size_t (*cbk_inodectx_size_t)(xlator_t *this, inode_t *inode);
+
+typedef size_t (*cbk_fdctx_size_t)(xlator_t *this, fd_t *fd);
+
struct xlator_cbks {
cbk_forget_t forget;
cbk_release_t release;
@@ -855,6 +859,8 @@ struct xlator_cbks {
cbk_client_t client_destroy;
cbk_client_t client_disconnect;
cbk_ictxmerge_t ictxmerge;
+ cbk_inodectx_size_t ictxsize;
+ cbk_fdctx_size_t fdctxsize;
};
typedef int32_t (*dumpop_priv_t) (xlator_t *this);
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 93ed1c8..cc95bf7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -2622,7 +2622,6 @@ struct volopt_map_entry glusterd_volopt_map[] = {
"option. Unmount and delete the shared storage volume "
" on disabling this option."
},
-
#if USE_GFDB /* no GFDB means tiering is disabled */
/* tier translator - global tunables */
{ .key = "cluster.write-freq-threshold",
@@ -3013,6 +3012,33 @@ struct volopt_map_entry glusterd_volopt_map[] = {
.op_version = GD_OP_VERSION_3_9_0,
.flags = OPT_FLAG_CLIENT_OPT
},
+ },
+ { .key = "performance.rda-request-size",
+ .voltype = "performance/readdir-ahead",
+ .option = "rda-request-size",
+ .flags = OPT_FLAG_CLIENT_OPT,
+ .type = DOC,
+ .op_version = GD_OP_VERSION_3_9_1,
+ },
+ { .key = "performance.rda-low-wmark",
+ .voltype = "performance/readdir-ahead",
+ .option = "rda-low-wmark",
+ .type = DOC,
+ .flags = OPT_FLAG_CLIENT_OPT,
+ .op_version = GD_OP_VERSION_3_9_1,
+ },
+ { .key = "performance.rda-high-wmark",
+ .voltype = "performance/readdir-ahead",
+ .type = DOC,
+ .flags = OPT_FLAG_CLIENT_OPT,
+ .op_version = GD_OP_VERSION_3_9_1,
+ },
+ { .key = "performance.rda-cache-limit",
+ .voltype = "performance/readdir-ahead",
+ .type = DOC,
+ .flags = OPT_FLAG_CLIENT_OPT,
+ .op_version = GD_OP_VERSION_3_9_1,
+ },
{ .key = NULL
}
};
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead.c b/xlators/performance/readdir-ahead/src/readdir-ahead.c
index c3daf91..4b57a8b 100644
--- a/xlators/performance/readdir-ahead/src/readdir-ahead.c
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead.c
@@ -97,7 +97,8 @@ rda_can_serve_readdirp(struct rda_fd_ctx *ctx, size_t request_size)
{
if ((ctx->state & RDA_FD_EOD) ||
(ctx->state & RDA_FD_ERROR) ||
- (!(ctx->state & RDA_FD_PLUGGED) && (ctx->cur_size > 0)))
+ (!(ctx->state & RDA_FD_PLUGGED) && (ctx->cur_size > 0)) ||
+ (request_size && ctx->cur_size >= request_size))
return _gf_true;
return _gf_false;
@@ -111,20 +112,28 @@ static int32_t
__rda_serve_readdirp(xlator_t *this, gf_dirent_t *entries, size_t request_size,
struct rda_fd_ctx *ctx)
{
- gf_dirent_t *dirent, *tmp;
- size_t dirent_size, size = 0;
- int32_t count = 0;
- struct rda_priv *priv = this->private;
+ gf_dirent_t *dirent, *tmp;
+ size_t dirent_size, size = 0, inodectx_size = 0;
+ int32_t count = 0;
+ struct rda_priv *priv = NULL;
+
+ priv = this->private;
list_for_each_entry_safe(dirent, tmp, &ctx->entries.list, list) {
dirent_size = gf_dirent_size(dirent->d_name);
if (size + dirent_size > request_size)
break;
+ inodectx_size = 0;
+
+ inode_ctx_del (dirent->inode, this, (void *)&inodectx_size);
+
size += dirent_size;
list_del_init(&dirent->list);
ctx->cur_size -= dirent_size;
+ priv->rda_cache_size -= (dirent_size + inodectx_size);
+
list_add_tail(&dirent->list, &entries->list);
ctx->cur_offset = dirent->d_off;
count++;
@@ -234,11 +243,17 @@ rda_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
* the request out of the preload or the request that enables us to do
* so is in flight...
*/
- if (rda_can_serve_readdirp(ctx, size))
+ if (rda_can_serve_readdirp(ctx, size)) {
call_resume(stub);
- else
+ } else {
ctx->stub = stub;
+ if (!(ctx->state & RDA_FD_RUNNING)) {
+ fill = 1;
+ ctx->state |= RDA_FD_RUNNING;
+ }
+ }
+
UNLOCK(&ctx->lock);
if (fill)
@@ -266,6 +281,7 @@ rda_fill_fd_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
struct rda_fd_ctx *ctx = local->ctx;
struct rda_priv *priv = this->private;
int fill = 1;
+ size_t inodectx_size = 0, dirent_size = 0;
LOCK(&ctx->lock);
@@ -286,7 +302,19 @@ rda_fill_fd_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
/* must preserve entry order */
list_add_tail(&dirent->list, &ctx->entries.list);
- ctx->cur_size += gf_dirent_size(dirent->d_name);
+ dirent_size = gf_dirent_size (dirent->d_name);
+ inodectx_size = 0;
+
+ if (dirent->inode) {
+ inodectx_size = inode_ctx_size (dirent->inode);
+ inode_ctx_set (dirent->inode, this,
+ (void *)inodectx_size);
+ }
+
+ ctx->cur_size += dirent_size;
+
+ priv->rda_cache_size += (dirent_size + inodectx_size);
+
ctx->next_offset = dirent->d_off;
}
}
@@ -321,19 +349,21 @@ out:
* If we have been marked for bypass and have no pending stub, clear the
* run state so we stop preloading the context with entries.
*/
- if ((ctx->state & RDA_FD_BYPASS) && !ctx->stub)
+ if (!ctx->stub && ((ctx->state & RDA_FD_BYPASS)
+ || (priv->rda_cache_size > priv->rda_cache_limit)))
ctx->state &= ~RDA_FD_RUNNING;
if (!(ctx->state & RDA_FD_RUNNING)) {
fill = 0;
- if (ctx->xattrs) {
- /*
- * fill = 0 and hence rda_fill_fd() won't be invoked.
- * unref for ref taken in rda_fill_fd()
- */
- dict_unref (ctx->xattrs);
- ctx->xattrs = NULL;
- }
+ if (ctx->xattrs) {
+ /*
+ * fill = 0 and hence rda_fill_fd() won't be invoked.
+ * unref for ref taken in rda_fill_fd()
+ */
+ dict_unref (ctx->xattrs);
+ ctx->xattrs = NULL;
+ }
+
STACK_DESTROY(ctx->fill_frame->root);
ctx->fill_frame = NULL;
}
@@ -393,10 +423,10 @@ rda_fill_fd(call_frame_t *frame, xlator_t *this, fd_t *fd)
ctx->fill_frame = nframe;
- if (!ctx->xattrs && orig_local && orig_local->xattrs) {
- /* when this function is invoked by rda_opendir_cbk */
- ctx->xattrs = dict_ref(orig_local->xattrs);
- }
+ if (!ctx->xattrs && orig_local && orig_local->xattrs) {
+ /* when this function is invoked by rda_opendir_cbk */
+ ctx->xattrs = dict_ref(orig_local->xattrs);
+ }
} else {
nframe = ctx->fill_frame;
local = nframe->local;
@@ -578,11 +608,13 @@ reconfigure(xlator_t *this, dict_t *options)
struct rda_priv *priv = this->private;
GF_OPTION_RECONF("rda-request-size", priv->rda_req_size, options,
- uint32, err);
- GF_OPTION_RECONF("rda-low-wmark", priv->rda_low_wmark, options, size_uint64,
- err);
- GF_OPTION_RECONF("rda-high-wmark", priv->rda_high_wmark, options, size_uint64,
- err);
+ size_uint64, err);
+ GF_OPTION_RECONF("rda-low-wmark", priv->rda_low_wmark, options,
+ size_uint64, err);
+ GF_OPTION_RECONF("rda-high-wmark", priv->rda_high_wmark, options,
+ size_uint64, err);
+ GF_OPTION_RECONF("rda-cache-limit", priv->rda_cache_limit, options,
+ size_uint64, err);
return 0;
err:
@@ -619,9 +651,13 @@ init(xlator_t *this)
if (!this->local_pool)
goto err;
- GF_OPTION_INIT("rda-request-size", priv->rda_req_size, uint32, err);
+ GF_OPTION_INIT("rda-request-size", priv->rda_req_size, size_uint64,
+ err);
GF_OPTION_INIT("rda-low-wmark", priv->rda_low_wmark, size_uint64, err);
- GF_OPTION_INIT("rda-high-wmark", priv->rda_high_wmark, size_uint64, err);
+ GF_OPTION_INIT("rda-high-wmark", priv->rda_high_wmark, size_uint64,
+ err);
+ GF_OPTION_INIT("rda-cache-limit", priv->rda_cache_limit, size_uint64,
+ err);
return 0;
@@ -657,26 +693,38 @@ struct xlator_cbks cbks = {
struct volume_options options[] = {
{ .key = {"rda-request-size"},
- .type = GF_OPTION_TYPE_INT,
+ .type = GF_OPTION_TYPE_SIZET,
.min = 4096,
.max = 131072,
- .default_value = "131072",
- .description = "readdir-ahead request size",
+ .default_value = "128KB",
+ .description = "size of buffer in readdirp calls initiated by "
+ "readdir-ahead ",
},
{ .key = {"rda-low-wmark"},
.type = GF_OPTION_TYPE_SIZET,
.min = 0,
.max = 10 * GF_UNIT_MB,
.default_value = "4096",
- .description = "the value under which we plug",
+ .description = "the value under which readdir-ahead plugs",
},
{ .key = {"rda-high-wmark"},
.type = GF_OPTION_TYPE_SIZET,
.min = 0,
.max = 100 * GF_UNIT_MB,
- .default_value = "131072",
- .description = "the value over which we unplug",
+ .default_value = "128KB",
+ .description = "the value over which readdir-ahead unplugs",
},
+ { .key = {"rda-cache-limit"},
+ .type = GF_OPTION_TYPE_SIZET,
+ .min = 0,
+ .max = 1 * GF_UNIT_GB,
+ .default_value = "10MB",
+ .description = "maximum size of cache consumed by readdir-ahead "
+ "xlator. This value is global and total memory "
+ "consumption by readdir-ahead is capped by this "
+ "value, irrespective of the number/size of "
+ "directories cached",
+ },
{ .key = {NULL} },
};
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead.h b/xlators/performance/readdir-ahead/src/readdir-ahead.h
index f030f10..6b65a62 100644
--- a/xlators/performance/readdir-ahead/src/readdir-ahead.h
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead.h
@@ -40,9 +40,11 @@ struct rda_local {
};
struct rda_priv {
- uint32_t rda_req_size;
+ uint64_t rda_req_size;
uint64_t rda_low_wmark;
uint64_t rda_high_wmark;
+ uint64_t rda_cache_limit;
+ uint64_t rda_cache_size;
};
#endif /* __READDIR_AHEAD_H */
--
2.9.3