From fe5617f5422ea052b17a4ab282f80ac6b61238df Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Mon, 17 Apr 2017 15:50:07 +0530 Subject: [PATCH 434/473] core: make the per glusterfs_ctx_t timer-wheel refcounted xlators can use a 'global' timer-wheel for scheduling events. This timer-wheel is managed per glusterfs_ctx_t, but does not need to be allocated for every graph. When an xlator wants to use the timer-wheel, it will be instanciated on demand, and provided to xlators that request it later on. By adding a reference counter to the glusterfs_ctx_t for the timer-wheel, the threads and structures can be cleaned up when the last xlator does not have a need for it anymore. In general, the xlators request the timer-wheel in init(), and they should return it in fini(). Because the timer-wheel is managed per glusterfs_ctx_t, the functions can be added to ctx.c and do not need to live in their very minimal tw.[ch] files. >Reported-by: Poornima G >Signed-off-by: Niels de Vos >Reviewed-on: https://review.gluster.org/17068 >NetBSD-regression: NetBSD Build System >CentOS-regression: Gluster Build System >Smoke: Gluster Build System >Reviewed-by: Amar Tumballi >Reviewed-by: Zhou Zhengping >Reviewed-by: Kaleb KEITHLEY >(cherry picked from commit 73fcf3a874b2049da31d01b8363d1ac85c9488c2) >Conflicts: > libglusterfs/src/Makefile.am Change-Id: I19d225b39aaa272d9005ba7adc3104c3764f1572 BUG: 1450336 Signed-off-by: Poornima G Reviewed-on: https://code.engineering.redhat.com/gerrit/106133 Reviewed-by: Atin Mukherjee --- glusterfsd/src/glusterfsd.c | 6 ++--- libglusterfs/src/Makefile.am | 6 ++--- libglusterfs/src/ctx.c | 39 ++++++++++++++++++++++++++++- libglusterfs/src/globals.h | 3 +++ libglusterfs/src/glusterfs.h | 18 ++++++++----- libglusterfs/src/mem-types.h | 1 + libglusterfs/src/tw.c | 25 ------------------ libglusterfs/src/tw.h | 23 ----------------- xlators/features/bit-rot/src/bitd/bit-rot.c | 6 ++--- xlators/features/leases/src/leases.c | 18 +++++-------- xlators/features/leases/src/leases.h | 1 - xlators/performance/nl-cache/src/nl-cache.c | 18 ++++++------- 12 files changed, 76 insertions(+), 88 deletions(-) delete mode 100644 libglusterfs/src/tw.c delete mode 100644 libglusterfs/src/tw.h diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c index 2724ed7..110cbb3 100644 --- a/glusterfsd/src/glusterfsd.c +++ b/glusterfsd/src/glusterfsd.c @@ -75,7 +75,6 @@ #include "exports.h" #include "daemon.h" -#include "tw.h" /* using argp for command line parsing */ @@ -2467,9 +2466,10 @@ main (int argc, char *argv[]) /* do this _after_ daemonize() */ if (cmd->global_timer_wheel) { - ret = glusterfs_global_timer_wheel_init (ctx); - if (ret) + if (!glusterfs_ctx_tw_get (ctx)) { + ret = -1; goto out; + } } ret = glusterfs_volumes_init (ctx); diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am index 01cb314..6abced9 100644 --- a/libglusterfs/src/Makefile.am +++ b/libglusterfs/src/Makefile.am @@ -32,8 +32,8 @@ libglusterfs_la_SOURCES = dict.c xlator.c logging.c \ strfd.c parse-utils.c $(CONTRIBDIR)/mount/mntent.c \ $(CONTRIBDIR)/libexecinfo/execinfo.c quota-common-utils.c rot-buffs.c \ $(CONTRIBDIR)/timer-wheel/timer-wheel.c \ - $(CONTRIBDIR)/timer-wheel/find_last_bit.c tw.c default-args.c locking.c \ - compound-fop-utils.c + $(CONTRIBDIR)/timer-wheel/find_last_bit.c default-args.c locking.c \ + compound-fop-utils.c throttle-tbf.c nodist_libglusterfs_la_SOURCES = y.tab.c graph.lex.c defaults.c nodist_libglusterfs_la_HEADERS = y.tab.h glusterfs-fops.h cli1-xdr.h @@ -50,7 +50,7 @@ libglusterfs_la_HEADERS = common-utils.h defaults.h default-args.h \ refcount.h run.h options.h lkowner.h fd-lk.h circ-buff.h \ event-history.h gidcache.h client_t.h glusterfs-acl.h \ glfs-message-id.h template-component-messages.h strfd.h \ - syncop-utils.h parse-utils.h libglusterfs-messages.h tw.h \ + syncop-utils.h parse-utils.h libglusterfs-messages.h \ lvm-defaults.h quota-common-utils.h rot-buffs.h \ compat-uuid.h upcall-utils.h events.h compound-fop-utils.h \ atomic.h diff --git a/libglusterfs/src/ctx.c b/libglusterfs/src/ctx.c index 2aa1465..1f2f1a1 100644 --- a/libglusterfs/src/ctx.c +++ b/libglusterfs/src/ctx.c @@ -9,9 +9,10 @@ */ #include -#include "globals.h" +#include "globals.h" #include "glusterfs.h" +#include "timer-wheel.h" glusterfs_ctx_t * glusterfs_ctx_new () @@ -46,3 +47,39 @@ out: return ctx; } +static void +glusterfs_ctx_tw_destroy (struct gf_ctx_tw *ctx_tw) +{ + if (ctx_tw->timer_wheel) + gf_tw_cleanup_timers (ctx_tw->timer_wheel); + + GF_FREE (ctx_tw); +} + +struct tvec_base* +glusterfs_ctx_tw_get (glusterfs_ctx_t *ctx) +{ + struct gf_ctx_tw *ctx_tw = NULL; + + LOCK (&ctx->lock); + { + if (ctx->tw) { + ctx_tw = GF_REF_GET (ctx->tw); + } else { + ctx_tw = GF_CALLOC (1, sizeof (struct gf_ctx_tw), + gf_common_mt_tw_ctx); + ctx_tw->timer_wheel = gf_tw_init_timers(); + GF_REF_INIT (ctx_tw, glusterfs_ctx_tw_destroy); + ctx->tw = ctx_tw; + } + } + UNLOCK (&ctx->lock); + + return ctx_tw->timer_wheel; +} + +void +glusterfs_ctx_tw_put (glusterfs_ctx_t *ctx) +{ + GF_REF_PUT (ctx->tw); +} diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h index 69c4c6d..08031a7 100644 --- a/libglusterfs/src/globals.h +++ b/libglusterfs/src/globals.h @@ -119,6 +119,9 @@ char *glusterfs_leaseid_buf_get (); /* init */ int glusterfs_globals_init (glusterfs_ctx_t *ctx); +struct tvec_base* glusterfs_ctx_tw_get (glusterfs_ctx_t *ctx); +void glusterfs_ctx_tw_put (glusterfs_ctx_t *ctx); + extern const char *gf_fop_list[]; extern const char *gf_upcall_list[]; diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h index f77f5f8..4e686c3 100644 --- a/libglusterfs/src/glusterfs.h +++ b/libglusterfs/src/glusterfs.h @@ -36,6 +36,7 @@ #include "logging.h" #include "lkowner.h" #include "compat-uuid.h" +#include "refcount.h" #define GF_YES 1 #define GF_NO 0 @@ -427,6 +428,12 @@ typedef enum { struct tvec_base; +/* reference counting for the global (per ctx) timer-wheel */ +struct gf_ctx_tw { + GF_REF_DECL; + struct tvec_base *timer_wheel; /* global timer-wheel instance */ +}; + struct _glusterfs_ctx { cmd_args_t cmd_args; char *process_uuid; @@ -495,14 +502,13 @@ struct _glusterfs_ctx { */ mgmt_ssl_t secure_srvr; /* Buffer to 'save' backtrace even under OOM-kill like situations*/ - char btbuf[GF_BACKTRACE_LEN]; + char btbuf[GF_BACKTRACE_LEN]; - pthread_mutex_t notify_lock; - pthread_cond_t notify_cond; - int notifying; - - struct tvec_base *timer_wheel; /* global timer-wheel instance */ + pthread_mutex_t notify_lock; + pthread_cond_t notify_cond; + int notifying; + struct gf_ctx_tw *tw; /* refcounted timer_wheel */ }; typedef struct _glusterfs_ctx glusterfs_ctx_t; diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h index afa52d8..ef7dcd0 100644 --- a/libglusterfs/src/mem-types.h +++ b/libglusterfs/src/mem-types.h @@ -164,6 +164,7 @@ enum gf_common_mem_types_ { /*used for compound fops*/ gf_mt_compound_req_t, gf_mt_compound_rsp_t, + gf_common_mt_tw_ctx, gf_common_mt_tw_timer_list, /*lock migration*/ gf_common_mt_lock_mig, diff --git a/libglusterfs/src/tw.c b/libglusterfs/src/tw.c deleted file mode 100644 index fa11998..0000000 --- a/libglusterfs/src/tw.c +++ /dev/null @@ -1,25 +0,0 @@ -/* - Copyright (c) 2008-2015 Red Hat, Inc. - This file is part of GlusterFS. - - This file is licensed to you under your choice of the GNU Lesser - General Public License, version 3 or any later version (LGPLv3 or - later), or the GNU General Public License, version 2 (GPLv2), in all - cases as published by the Free Software Foundation. -*/ - -#include "tw.h" -#include "timer-wheel.h" - -int -glusterfs_global_timer_wheel_init (glusterfs_ctx_t *ctx) -{ - ctx->timer_wheel = gf_tw_init_timers(); - return ctx->timer_wheel ? 0 : -1; -} - -struct tvec_base * -glusterfs_global_timer_wheel (xlator_t *this) -{ - return this->ctx->timer_wheel; -} diff --git a/libglusterfs/src/tw.h b/libglusterfs/src/tw.h deleted file mode 100644 index e635cd2..0000000 --- a/libglusterfs/src/tw.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - Copyright (c) 2008-2015 Red Hat, Inc. - This file is part of GlusterFS. - - This file is licensed to you under your choice of the GNU Lesser - General Public License, version 3 or any later version (LGPLv3 or - later), or the GNU General Public License, version 2 (GPLv2), in all - cases as published by the Free Software Foundation. -*/ - -#ifndef __TW_H__ -#define __TW_H__ - -#include "xlator.h" -#include "glusterfs.h" - -int -glusterfs_global_timer_wheel_init (glusterfs_ctx_t *); - -struct tvec_base * -glusterfs_global_timer_wheel (xlator_t *); - -#endif /* __TW_H__ */ diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c index ab32925..6d60ca4 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot.c +++ b/xlators/features/bit-rot/src/bitd/bit-rot.c @@ -21,8 +21,6 @@ #include #include "bit-rot-bitd-messages.h" -#include "tw.h" - #define BR_HASH_CALC_READ_SIZE (128 * 1024) typedef int32_t (br_child_handler)(xlator_t *, br_child_t *); @@ -1976,7 +1974,7 @@ init (xlator_t *this) INIT_LIST_HEAD (&priv->bricks); INIT_LIST_HEAD (&priv->signing); - priv->timer_wheel = glusterfs_global_timer_wheel (this); + priv->timer_wheel = glusterfs_ctx_tw_get (this->ctx); if (!priv->timer_wheel) { gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_TIMER_WHEEL_UNAVAILABLE, @@ -2044,6 +2042,8 @@ fini (xlator_t *this) this->private = NULL; GF_FREE (priv); + glusterfs_ctx_tw_put (this->ctx); + return; } diff --git a/xlators/features/leases/src/leases.c b/xlators/features/leases/src/leases.c index 3e04600..faffa0e 100644 --- a/xlators/features/leases/src/leases.c +++ b/xlators/features/leases/src/leases.c @@ -957,19 +957,11 @@ leases_init_priv (xlator_t *this) GF_ASSERT (priv); if (!priv->timer_wheel) { - if (!glusterfs_global_timer_wheel (this)) { - gf_msg_debug (this->name, 0, "Initing the global " - "timer wheel"); - ret = glusterfs_global_timer_wheel_init (this->ctx); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, - LEASE_MSG_NO_TIMER_WHEEL, - "Initing the global timer " - "wheel failed"); - goto out; - } + priv->timer_wheel = glusterfs_ctx_tw_get (this->ctx); + if (!priv->timer_wheel) { + ret = -1; + goto out; } - priv->timer_wheel = glusterfs_global_timer_wheel (this); } if (!priv->inited_recall_thr) { @@ -1076,6 +1068,8 @@ fini (xlator_t *this) GF_FREE (priv); + glusterfs_ctx_tw_put (this->ctx); + return 0; } diff --git a/xlators/features/leases/src/leases.h b/xlators/features/leases/src/leases.h index df5e8be..a6cc34f 100644 --- a/xlators/features/leases/src/leases.h +++ b/xlators/features/leases/src/leases.h @@ -26,7 +26,6 @@ #include "lkowner.h" #include "locking.h" #include "upcall-utils.h" -#include "tw.h" #include "timer-wheel.h" #include "leases-mem-types.h" #include "leases-messages.h" diff --git a/xlators/performance/nl-cache/src/nl-cache.c b/xlators/performance/nl-cache/src/nl-cache.c index a34b752..92dbccb 100644 --- a/xlators/performance/nl-cache/src/nl-cache.c +++ b/xlators/performance/nl-cache/src/nl-cache.c @@ -12,7 +12,6 @@ #include "nl-cache.h" #include "statedump.h" #include "upcall-utils.h" -#include "tw.h" static void nlc_dentry_op (call_frame_t *frame, xlator_t *this, gf_boolean_t multilink) @@ -625,6 +624,8 @@ nlc_priv_dump (xlator_t *this) void fini (xlator_t *this) { + glusterfs_ctx_tw_put (this->ctx); + return; } @@ -700,17 +701,12 @@ init (xlator_t *this) INIT_LIST_HEAD (&conf->lru); time (&conf->last_child_down); - if (!glusterfs_global_timer_wheel (this)) { - gf_msg_debug (this->name, 0, "Initing the global timer wheel"); - ret = glusterfs_global_timer_wheel_init (this->ctx); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, - NLC_MSG_NO_TIMER_WHEEL, - "Initing the global timer wheel failed"); - goto out; - } + conf->timer_wheel = glusterfs_ctx_tw_get (this->ctx); + if (!conf->timer_wheel) { + gf_msg (this->name, GF_LOG_ERROR, 0, NLC_MSG_NO_TIMER_WHEEL, + "Initing the global timer wheel failed"); + goto out; } - conf->timer_wheel = glusterfs_global_timer_wheel (this); this->private = conf; -- 1.8.3.1