From fe5617f5422ea052b17a4ab282f80ac6b61238df Mon Sep 17 00:00:00 2001
From: Niels de Vos <ndevos@redhat.com>
Date: Mon, 17 Apr 2017 15:50:07 +0530
Subject: [PATCH 434/473] core: make the per glusterfs_ctx_t timer-wheel
refcounted
xlators can use a 'global' timer-wheel for scheduling events. This
timer-wheel is managed per glusterfs_ctx_t, but does not need to be
allocated for every graph. When an xlator wants to use the timer-wheel,
it will be instanciated on demand, and provided to xlators that request
it later on.
By adding a reference counter to the glusterfs_ctx_t for the
timer-wheel, the threads and structures can be cleaned up when the last
xlator does not have a need for it anymore. In general, the xlators
request the timer-wheel in init(), and they should return it in fini().
Because the timer-wheel is managed per glusterfs_ctx_t, the functions
can be added to ctx.c and do not need to live in their very minimal
tw.[ch] files.
>Reported-by: Poornima G <pgurusid@redhat.com>
>Signed-off-by: Niels de Vos <ndevos@redhat.com>
>Reviewed-on: https://review.gluster.org/17068
>NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
>CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
>Smoke: Gluster Build System <jenkins@build.gluster.org>
>Reviewed-by: Amar Tumballi <amarts@redhat.com>
>Reviewed-by: Zhou Zhengping <johnzzpcrystal@gmail.com>
>Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
>(cherry picked from commit 73fcf3a874b2049da31d01b8363d1ac85c9488c2)
>Conflicts:
> libglusterfs/src/Makefile.am
Change-Id: I19d225b39aaa272d9005ba7adc3104c3764f1572
BUG: 1450336
Signed-off-by: Poornima G <pgurusid@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/106133
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
---
glusterfsd/src/glusterfsd.c | 6 ++---
libglusterfs/src/Makefile.am | 6 ++---
libglusterfs/src/ctx.c | 39 ++++++++++++++++++++++++++++-
libglusterfs/src/globals.h | 3 +++
libglusterfs/src/glusterfs.h | 18 ++++++++-----
libglusterfs/src/mem-types.h | 1 +
libglusterfs/src/tw.c | 25 ------------------
libglusterfs/src/tw.h | 23 -----------------
xlators/features/bit-rot/src/bitd/bit-rot.c | 6 ++---
xlators/features/leases/src/leases.c | 18 +++++--------
xlators/features/leases/src/leases.h | 1 -
xlators/performance/nl-cache/src/nl-cache.c | 18 ++++++-------
12 files changed, 76 insertions(+), 88 deletions(-)
delete mode 100644 libglusterfs/src/tw.c
delete mode 100644 libglusterfs/src/tw.h
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 2724ed7..110cbb3 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -75,7 +75,6 @@
#include "exports.h"
#include "daemon.h"
-#include "tw.h"
/* using argp for command line parsing */
@@ -2467,9 +2466,10 @@ main (int argc, char *argv[])
/* do this _after_ daemonize() */
if (cmd->global_timer_wheel) {
- ret = glusterfs_global_timer_wheel_init (ctx);
- if (ret)
+ if (!glusterfs_ctx_tw_get (ctx)) {
+ ret = -1;
goto out;
+ }
}
ret = glusterfs_volumes_init (ctx);
diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am
index 01cb314..6abced9 100644
--- a/libglusterfs/src/Makefile.am
+++ b/libglusterfs/src/Makefile.am
@@ -32,8 +32,8 @@ libglusterfs_la_SOURCES = dict.c xlator.c logging.c \
strfd.c parse-utils.c $(CONTRIBDIR)/mount/mntent.c \
$(CONTRIBDIR)/libexecinfo/execinfo.c quota-common-utils.c rot-buffs.c \
$(CONTRIBDIR)/timer-wheel/timer-wheel.c \
- $(CONTRIBDIR)/timer-wheel/find_last_bit.c tw.c default-args.c locking.c \
- compound-fop-utils.c
+ $(CONTRIBDIR)/timer-wheel/find_last_bit.c default-args.c locking.c \
+ compound-fop-utils.c throttle-tbf.c
nodist_libglusterfs_la_SOURCES = y.tab.c graph.lex.c defaults.c
nodist_libglusterfs_la_HEADERS = y.tab.h glusterfs-fops.h cli1-xdr.h
@@ -50,7 +50,7 @@ libglusterfs_la_HEADERS = common-utils.h defaults.h default-args.h \
refcount.h run.h options.h lkowner.h fd-lk.h circ-buff.h \
event-history.h gidcache.h client_t.h glusterfs-acl.h \
glfs-message-id.h template-component-messages.h strfd.h \
- syncop-utils.h parse-utils.h libglusterfs-messages.h tw.h \
+ syncop-utils.h parse-utils.h libglusterfs-messages.h \
lvm-defaults.h quota-common-utils.h rot-buffs.h \
compat-uuid.h upcall-utils.h events.h compound-fop-utils.h \
atomic.h
diff --git a/libglusterfs/src/ctx.c b/libglusterfs/src/ctx.c
index 2aa1465..1f2f1a1 100644
--- a/libglusterfs/src/ctx.c
+++ b/libglusterfs/src/ctx.c
@@ -9,9 +9,10 @@
*/
#include <pthread.h>
-#include "globals.h"
+#include "globals.h"
#include "glusterfs.h"
+#include "timer-wheel.h"
glusterfs_ctx_t *
glusterfs_ctx_new ()
@@ -46,3 +47,39 @@ out:
return ctx;
}
+static void
+glusterfs_ctx_tw_destroy (struct gf_ctx_tw *ctx_tw)
+{
+ if (ctx_tw->timer_wheel)
+ gf_tw_cleanup_timers (ctx_tw->timer_wheel);
+
+ GF_FREE (ctx_tw);
+}
+
+struct tvec_base*
+glusterfs_ctx_tw_get (glusterfs_ctx_t *ctx)
+{
+ struct gf_ctx_tw *ctx_tw = NULL;
+
+ LOCK (&ctx->lock);
+ {
+ if (ctx->tw) {
+ ctx_tw = GF_REF_GET (ctx->tw);
+ } else {
+ ctx_tw = GF_CALLOC (1, sizeof (struct gf_ctx_tw),
+ gf_common_mt_tw_ctx);
+ ctx_tw->timer_wheel = gf_tw_init_timers();
+ GF_REF_INIT (ctx_tw, glusterfs_ctx_tw_destroy);
+ ctx->tw = ctx_tw;
+ }
+ }
+ UNLOCK (&ctx->lock);
+
+ return ctx_tw->timer_wheel;
+}
+
+void
+glusterfs_ctx_tw_put (glusterfs_ctx_t *ctx)
+{
+ GF_REF_PUT (ctx->tw);
+}
diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
index 69c4c6d..08031a7 100644
--- a/libglusterfs/src/globals.h
+++ b/libglusterfs/src/globals.h
@@ -119,6 +119,9 @@ char *glusterfs_leaseid_buf_get ();
/* init */
int glusterfs_globals_init (glusterfs_ctx_t *ctx);
+struct tvec_base* glusterfs_ctx_tw_get (glusterfs_ctx_t *ctx);
+void glusterfs_ctx_tw_put (glusterfs_ctx_t *ctx);
+
extern const char *gf_fop_list[];
extern const char *gf_upcall_list[];
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index f77f5f8..4e686c3 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -36,6 +36,7 @@
#include "logging.h"
#include "lkowner.h"
#include "compat-uuid.h"
+#include "refcount.h"
#define GF_YES 1
#define GF_NO 0
@@ -427,6 +428,12 @@ typedef enum {
struct tvec_base;
+/* reference counting for the global (per ctx) timer-wheel */
+struct gf_ctx_tw {
+ GF_REF_DECL;
+ struct tvec_base *timer_wheel; /* global timer-wheel instance */
+};
+
struct _glusterfs_ctx {
cmd_args_t cmd_args;
char *process_uuid;
@@ -495,14 +502,13 @@ struct _glusterfs_ctx {
*/
mgmt_ssl_t secure_srvr;
/* Buffer to 'save' backtrace even under OOM-kill like situations*/
- char btbuf[GF_BACKTRACE_LEN];
+ char btbuf[GF_BACKTRACE_LEN];
- pthread_mutex_t notify_lock;
- pthread_cond_t notify_cond;
- int notifying;
-
- struct tvec_base *timer_wheel; /* global timer-wheel instance */
+ pthread_mutex_t notify_lock;
+ pthread_cond_t notify_cond;
+ int notifying;
+ struct gf_ctx_tw *tw; /* refcounted timer_wheel */
};
typedef struct _glusterfs_ctx glusterfs_ctx_t;
diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h
index afa52d8..ef7dcd0 100644
--- a/libglusterfs/src/mem-types.h
+++ b/libglusterfs/src/mem-types.h
@@ -164,6 +164,7 @@ enum gf_common_mem_types_ {
/*used for compound fops*/
gf_mt_compound_req_t,
gf_mt_compound_rsp_t,
+ gf_common_mt_tw_ctx,
gf_common_mt_tw_timer_list,
/*lock migration*/
gf_common_mt_lock_mig,
diff --git a/libglusterfs/src/tw.c b/libglusterfs/src/tw.c
deleted file mode 100644
index fa11998..0000000
--- a/libglusterfs/src/tw.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
- This file is part of GlusterFS.
-
- This file is licensed to you under your choice of the GNU Lesser
- General Public License, version 3 or any later version (LGPLv3 or
- later), or the GNU General Public License, version 2 (GPLv2), in all
- cases as published by the Free Software Foundation.
-*/
-
-#include "tw.h"
-#include "timer-wheel.h"
-
-int
-glusterfs_global_timer_wheel_init (glusterfs_ctx_t *ctx)
-{
- ctx->timer_wheel = gf_tw_init_timers();
- return ctx->timer_wheel ? 0 : -1;
-}
-
-struct tvec_base *
-glusterfs_global_timer_wheel (xlator_t *this)
-{
- return this->ctx->timer_wheel;
-}
diff --git a/libglusterfs/src/tw.h b/libglusterfs/src/tw.h
deleted file mode 100644
index e635cd2..0000000
--- a/libglusterfs/src/tw.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
- This file is part of GlusterFS.
-
- This file is licensed to you under your choice of the GNU Lesser
- General Public License, version 3 or any later version (LGPLv3 or
- later), or the GNU General Public License, version 2 (GPLv2), in all
- cases as published by the Free Software Foundation.
-*/
-
-#ifndef __TW_H__
-#define __TW_H__
-
-#include "xlator.h"
-#include "glusterfs.h"
-
-int
-glusterfs_global_timer_wheel_init (glusterfs_ctx_t *);
-
-struct tvec_base *
-glusterfs_global_timer_wheel (xlator_t *);
-
-#endif /* __TW_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c
index ab32925..6d60ca4 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.c
@@ -21,8 +21,6 @@
#include <pthread.h>
#include "bit-rot-bitd-messages.h"
-#include "tw.h"
-
#define BR_HASH_CALC_READ_SIZE (128 * 1024)
typedef int32_t (br_child_handler)(xlator_t *, br_child_t *);
@@ -1976,7 +1974,7 @@ init (xlator_t *this)
INIT_LIST_HEAD (&priv->bricks);
INIT_LIST_HEAD (&priv->signing);
- priv->timer_wheel = glusterfs_global_timer_wheel (this);
+ priv->timer_wheel = glusterfs_ctx_tw_get (this->ctx);
if (!priv->timer_wheel) {
gf_msg (this->name, GF_LOG_ERROR, 0,
BRB_MSG_TIMER_WHEEL_UNAVAILABLE,
@@ -2044,6 +2042,8 @@ fini (xlator_t *this)
this->private = NULL;
GF_FREE (priv);
+ glusterfs_ctx_tw_put (this->ctx);
+
return;
}
diff --git a/xlators/features/leases/src/leases.c b/xlators/features/leases/src/leases.c
index 3e04600..faffa0e 100644
--- a/xlators/features/leases/src/leases.c
+++ b/xlators/features/leases/src/leases.c
@@ -957,19 +957,11 @@ leases_init_priv (xlator_t *this)
GF_ASSERT (priv);
if (!priv->timer_wheel) {
- if (!glusterfs_global_timer_wheel (this)) {
- gf_msg_debug (this->name, 0, "Initing the global "
- "timer wheel");
- ret = glusterfs_global_timer_wheel_init (this->ctx);
- if (ret) {
- gf_msg (this->name, GF_LOG_ERROR, 0,
- LEASE_MSG_NO_TIMER_WHEEL,
- "Initing the global timer "
- "wheel failed");
- goto out;
- }
+ priv->timer_wheel = glusterfs_ctx_tw_get (this->ctx);
+ if (!priv->timer_wheel) {
+ ret = -1;
+ goto out;
}
- priv->timer_wheel = glusterfs_global_timer_wheel (this);
}
if (!priv->inited_recall_thr) {
@@ -1076,6 +1068,8 @@ fini (xlator_t *this)
GF_FREE (priv);
+ glusterfs_ctx_tw_put (this->ctx);
+
return 0;
}
diff --git a/xlators/features/leases/src/leases.h b/xlators/features/leases/src/leases.h
index df5e8be..a6cc34f 100644
--- a/xlators/features/leases/src/leases.h
+++ b/xlators/features/leases/src/leases.h
@@ -26,7 +26,6 @@
#include "lkowner.h"
#include "locking.h"
#include "upcall-utils.h"
-#include "tw.h"
#include "timer-wheel.h"
#include "leases-mem-types.h"
#include "leases-messages.h"
diff --git a/xlators/performance/nl-cache/src/nl-cache.c b/xlators/performance/nl-cache/src/nl-cache.c
index a34b752..92dbccb 100644
--- a/xlators/performance/nl-cache/src/nl-cache.c
+++ b/xlators/performance/nl-cache/src/nl-cache.c
@@ -12,7 +12,6 @@
#include "nl-cache.h"
#include "statedump.h"
#include "upcall-utils.h"
-#include "tw.h"
static void
nlc_dentry_op (call_frame_t *frame, xlator_t *this, gf_boolean_t multilink)
@@ -625,6 +624,8 @@ nlc_priv_dump (xlator_t *this)
void
fini (xlator_t *this)
{
+ glusterfs_ctx_tw_put (this->ctx);
+
return;
}
@@ -700,17 +701,12 @@ init (xlator_t *this)
INIT_LIST_HEAD (&conf->lru);
time (&conf->last_child_down);
- if (!glusterfs_global_timer_wheel (this)) {
- gf_msg_debug (this->name, 0, "Initing the global timer wheel");
- ret = glusterfs_global_timer_wheel_init (this->ctx);
- if (ret) {
- gf_msg (this->name, GF_LOG_ERROR, 0,
- NLC_MSG_NO_TIMER_WHEEL,
- "Initing the global timer wheel failed");
- goto out;
- }
+ conf->timer_wheel = glusterfs_ctx_tw_get (this->ctx);
+ if (!conf->timer_wheel) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, NLC_MSG_NO_TIMER_WHEEL,
+ "Initing the global timer wheel failed");
+ goto out;
}
- conf->timer_wheel = glusterfs_global_timer_wheel (this);
this->private = conf;
--
1.8.3.1