From fe5617f5422ea052b17a4ab282f80ac6b61238df Mon Sep 17 00:00:00 2001

From: Niels de Vos <ndevos@redhat.com>

Date: Mon, 17 Apr 2017 15:50:07 +0530

Subject: [PATCH 434/473] core: make the per glusterfs_ctx_t timer-wheel

 refcounted

xlators can use a 'global' timer-wheel for scheduling events. This

timer-wheel is managed per glusterfs_ctx_t, but does not need to be

allocated for every graph. When an xlator wants to use the timer-wheel,

it will be instanciated on demand, and provided to xlators that request

it later on.

By adding a reference counter to the glusterfs_ctx_t for the

timer-wheel, the threads and structures can be cleaned up when the last

xlator does not have a need for it anymore. In general, the xlators

request the timer-wheel in init(), and they should return it in fini().

Because the timer-wheel is managed per glusterfs_ctx_t, the functions

can be added to ctx.c and do not need to live in their very minimal

tw.[ch] files.

>Reported-by: Poornima G <pgurusid@redhat.com>

>Signed-off-by: Niels de Vos <ndevos@redhat.com>

>Reviewed-on: https://review.gluster.org/17068

>NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

>CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

>Smoke: Gluster Build System <jenkins@build.gluster.org>

>Reviewed-by: Amar Tumballi <amarts@redhat.com>

>Reviewed-by: Zhou Zhengping <johnzzpcrystal@gmail.com>

>Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

>(cherry picked from commit 73fcf3a874b2049da31d01b8363d1ac85c9488c2)

>Conflicts:

>        libglusterfs/src/Makefile.am

Change-Id: I19d225b39aaa272d9005ba7adc3104c3764f1572

BUG: 1450336

Signed-off-by: Poornima G <pgurusid@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/106133

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

---

 glusterfsd/src/glusterfsd.c                 |  6 ++---

 libglusterfs/src/Makefile.am                |  6 ++---

 libglusterfs/src/ctx.c                      | 39 ++++++++++++++++++++++++++++-

 libglusterfs/src/globals.h                  |  3 +++

 libglusterfs/src/glusterfs.h                | 18 ++++++++-----

 libglusterfs/src/mem-types.h                |  1 +

 libglusterfs/src/tw.c                       | 25 ------------------

 libglusterfs/src/tw.h                       | 23 -----------------

 xlators/features/bit-rot/src/bitd/bit-rot.c |  6 ++---

 xlators/features/leases/src/leases.c        | 18 +++++--------

 xlators/features/leases/src/leases.h        |  1 -

 xlators/performance/nl-cache/src/nl-cache.c | 18 ++++++-------

 12 files changed, 76 insertions(+), 88 deletions(-)

 delete mode 100644 libglusterfs/src/tw.c

 delete mode 100644 libglusterfs/src/tw.h

diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c

index 2724ed7..110cbb3 100644

--- a/glusterfsd/src/glusterfsd.c

+++ b/glusterfsd/src/glusterfsd.c

@@ -75,7 +75,6 @@

 #include "exports.h"

 #include "daemon.h"

-#include "tw.h"

 /* using argp for command line parsing */

@@ -2467,9 +2466,10 @@ main (int argc, char *argv[])

         /* do this _after_ daemonize() */

         if (cmd->global_timer_wheel) {

-                ret = glusterfs_global_timer_wheel_init (ctx);

-                if (ret)

+                if (!glusterfs_ctx_tw_get (ctx)) {

+                        ret = -1;

                         goto out;

+                }

         }

         ret = glusterfs_volumes_init (ctx);

diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am

index 01cb314..6abced9 100644

--- a/libglusterfs/src/Makefile.am

+++ b/libglusterfs/src/Makefile.am

@@ -32,8 +32,8 @@ libglusterfs_la_SOURCES = dict.c xlator.c logging.c \

 	strfd.c parse-utils.c $(CONTRIBDIR)/mount/mntent.c \

 	$(CONTRIBDIR)/libexecinfo/execinfo.c quota-common-utils.c rot-buffs.c \

 	$(CONTRIBDIR)/timer-wheel/timer-wheel.c \

-	$(CONTRIBDIR)/timer-wheel/find_last_bit.c tw.c default-args.c locking.c \

-	compound-fop-utils.c

+	$(CONTRIBDIR)/timer-wheel/find_last_bit.c default-args.c locking.c \

+	compound-fop-utils.c throttle-tbf.c

 nodist_libglusterfs_la_SOURCES = y.tab.c graph.lex.c defaults.c

 nodist_libglusterfs_la_HEADERS = y.tab.h glusterfs-fops.h cli1-xdr.h

@@ -50,7 +50,7 @@ libglusterfs_la_HEADERS = common-utils.h defaults.h default-args.h \

 	refcount.h run.h options.h lkowner.h fd-lk.h circ-buff.h \

 	event-history.h gidcache.h client_t.h glusterfs-acl.h \

 	glfs-message-id.h template-component-messages.h strfd.h \

-	syncop-utils.h parse-utils.h libglusterfs-messages.h tw.h \

+	syncop-utils.h parse-utils.h libglusterfs-messages.h \

 	lvm-defaults.h quota-common-utils.h rot-buffs.h \

 	compat-uuid.h upcall-utils.h events.h compound-fop-utils.h \

         atomic.h

diff --git a/libglusterfs/src/ctx.c b/libglusterfs/src/ctx.c

index 2aa1465..1f2f1a1 100644

--- a/libglusterfs/src/ctx.c

+++ b/libglusterfs/src/ctx.c

@@ -9,9 +9,10 @@

 */

 #include <pthread.h>

-#include "globals.h"

+#include "globals.h"

 #include "glusterfs.h"

+#include "timer-wheel.h"

 glusterfs_ctx_t *

 glusterfs_ctx_new ()

@@ -46,3 +47,39 @@ out:

 	return ctx;

 }

+static void

+glusterfs_ctx_tw_destroy (struct gf_ctx_tw *ctx_tw)

+{

+        if (ctx_tw->timer_wheel)

+                gf_tw_cleanup_timers (ctx_tw->timer_wheel);

+

+        GF_FREE (ctx_tw);

+}

+

+struct tvec_base*

+glusterfs_ctx_tw_get (glusterfs_ctx_t *ctx)

+{

+        struct gf_ctx_tw *ctx_tw = NULL;

+

+        LOCK (&ctx->lock);

+        {

+                if (ctx->tw) {

+                        ctx_tw = GF_REF_GET (ctx->tw);

+                } else {

+                        ctx_tw = GF_CALLOC (1, sizeof (struct gf_ctx_tw),

+                                            gf_common_mt_tw_ctx);

+                        ctx_tw->timer_wheel = gf_tw_init_timers();

+                        GF_REF_INIT (ctx_tw, glusterfs_ctx_tw_destroy);

+                        ctx->tw = ctx_tw;

+                }

+        }

+        UNLOCK (&ctx->lock);

+

+        return ctx_tw->timer_wheel;

+}

+

+void

+glusterfs_ctx_tw_put (glusterfs_ctx_t *ctx)

+{

+        GF_REF_PUT (ctx->tw);

+}

diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h

index 69c4c6d..08031a7 100644

--- a/libglusterfs/src/globals.h

+++ b/libglusterfs/src/globals.h

@@ -119,6 +119,9 @@ char *glusterfs_leaseid_buf_get ();

 /* init */

 int glusterfs_globals_init (glusterfs_ctx_t *ctx);

+struct tvec_base* glusterfs_ctx_tw_get (glusterfs_ctx_t *ctx);

+void glusterfs_ctx_tw_put (glusterfs_ctx_t *ctx);

+

 extern const char *gf_fop_list[];

 extern const char *gf_upcall_list[];

diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h

index f77f5f8..4e686c3 100644

--- a/libglusterfs/src/glusterfs.h

+++ b/libglusterfs/src/glusterfs.h

@@ -36,6 +36,7 @@

 #include "logging.h"

 #include "lkowner.h"

 #include "compat-uuid.h"

+#include "refcount.h"

 #define GF_YES 1

 #define GF_NO  0

@@ -427,6 +428,12 @@ typedef enum {

 struct tvec_base;

+/* reference counting for the global (per ctx) timer-wheel */

+struct gf_ctx_tw {

+        GF_REF_DECL;

+        struct tvec_base *timer_wheel; /* global timer-wheel instance */

+};

+

 struct _glusterfs_ctx {

         cmd_args_t          cmd_args;

         char               *process_uuid;

@@ -495,14 +502,13 @@ struct _glusterfs_ctx {

          */

         mgmt_ssl_t          secure_srvr;

         /* Buffer to 'save' backtrace even under OOM-kill like situations*/

-        char btbuf[GF_BACKTRACE_LEN];

+        char                btbuf[GF_BACKTRACE_LEN];

-        pthread_mutex_t notify_lock;

-        pthread_cond_t notify_cond;

-        int notifying;

-

-        struct tvec_base *timer_wheel; /* global timer-wheel instance */

+        pthread_mutex_t     notify_lock;

+        pthread_cond_t      notify_cond;

+        int                 notifying;

+        struct gf_ctx_tw   *tw; /* refcounted timer_wheel */

 };

 typedef struct _glusterfs_ctx glusterfs_ctx_t;

diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h

index afa52d8..ef7dcd0 100644

--- a/libglusterfs/src/mem-types.h

+++ b/libglusterfs/src/mem-types.h

@@ -164,6 +164,7 @@ enum gf_common_mem_types_ {

         /*used for compound fops*/

         gf_mt_compound_req_t,

         gf_mt_compound_rsp_t,

+        gf_common_mt_tw_ctx,

         gf_common_mt_tw_timer_list,

         /*lock migration*/

         gf_common_mt_lock_mig,

diff --git a/libglusterfs/src/tw.c b/libglusterfs/src/tw.c

deleted file mode 100644

index fa11998..0000000

--- a/libglusterfs/src/tw.c

+++ /dev/null

@@ -1,25 +0,0 @@

-/*

-  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>

-  This file is part of GlusterFS.

-

-  This file is licensed to you under your choice of the GNU Lesser

-  General Public License, version 3 or any later version (LGPLv3 or

-  later), or the GNU General Public License, version 2 (GPLv2), in all

-  cases as published by the Free Software Foundation.

-*/

-

-#include "tw.h"

-#include "timer-wheel.h"

-

-int

-glusterfs_global_timer_wheel_init (glusterfs_ctx_t *ctx)

-{

-        ctx->timer_wheel = gf_tw_init_timers();

-        return ctx->timer_wheel ? 0 : -1;

-}

-

-struct tvec_base *

-glusterfs_global_timer_wheel (xlator_t *this)

-{

-        return this->ctx->timer_wheel;

-}

diff --git a/libglusterfs/src/tw.h b/libglusterfs/src/tw.h

deleted file mode 100644

index e635cd2..0000000

--- a/libglusterfs/src/tw.h

+++ /dev/null

@@ -1,23 +0,0 @@

-/*

-  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>

-  This file is part of GlusterFS.

-

-  This file is licensed to you under your choice of the GNU Lesser

-  General Public License, version 3 or any later version (LGPLv3 or

-  later), or the GNU General Public License, version 2 (GPLv2), in all

-  cases as published by the Free Software Foundation.

-*/

-

-#ifndef __TW_H__

-#define __TW_H__

-

-#include "xlator.h"

-#include "glusterfs.h"

-

-int

-glusterfs_global_timer_wheel_init (glusterfs_ctx_t *);

-

-struct tvec_base *

-glusterfs_global_timer_wheel (xlator_t *);

-

-#endif /* __TW_H__ */

diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c

index ab32925..6d60ca4 100644

--- a/xlators/features/bit-rot/src/bitd/bit-rot.c

+++ b/xlators/features/bit-rot/src/bitd/bit-rot.c

@@ -21,8 +21,6 @@

 #include <pthread.h>

 #include "bit-rot-bitd-messages.h"

-#include "tw.h"

-

 #define BR_HASH_CALC_READ_SIZE  (128 * 1024)

 typedef int32_t (br_child_handler)(xlator_t *, br_child_t *);

@@ -1976,7 +1974,7 @@ init (xlator_t *this)

         INIT_LIST_HEAD (&priv->bricks);

         INIT_LIST_HEAD (&priv->signing);

-        priv->timer_wheel = glusterfs_global_timer_wheel (this);

+        priv->timer_wheel = glusterfs_ctx_tw_get (this->ctx);

         if (!priv->timer_wheel) {

                 gf_msg (this->name, GF_LOG_ERROR, 0,

                         BRB_MSG_TIMER_WHEEL_UNAVAILABLE,

@@ -2044,6 +2042,8 @@ fini (xlator_t *this)

         this->private = NULL;

 	GF_FREE (priv);

+        glusterfs_ctx_tw_put (this->ctx);

+

 	return;

 }

diff --git a/xlators/features/leases/src/leases.c b/xlators/features/leases/src/leases.c

index 3e04600..faffa0e 100644

--- a/xlators/features/leases/src/leases.c

+++ b/xlators/features/leases/src/leases.c

@@ -957,19 +957,11 @@ leases_init_priv (xlator_t *this)

         GF_ASSERT (priv);

         if (!priv->timer_wheel) {

-                if (!glusterfs_global_timer_wheel (this)) {

-                        gf_msg_debug (this->name, 0, "Initing the global "

-                                      "timer wheel");

-                        ret = glusterfs_global_timer_wheel_init (this->ctx);

-                        if (ret) {

-                                gf_msg (this->name, GF_LOG_ERROR, 0,

-                                        LEASE_MSG_NO_TIMER_WHEEL,

-                                        "Initing the global timer "

-                                        "wheel failed");

-                                goto out;

-                        }

+                priv->timer_wheel = glusterfs_ctx_tw_get (this->ctx);

+                if (!priv->timer_wheel) {

+                        ret = -1;

+                        goto out;

                 }

-                priv->timer_wheel = glusterfs_global_timer_wheel (this);

         }

         if (!priv->inited_recall_thr) {

@@ -1076,6 +1068,8 @@ fini (xlator_t *this)

         GF_FREE (priv);

+        glusterfs_ctx_tw_put (this->ctx);

+

         return 0;

 }

diff --git a/xlators/features/leases/src/leases.h b/xlators/features/leases/src/leases.h

index df5e8be..a6cc34f 100644

--- a/xlators/features/leases/src/leases.h

+++ b/xlators/features/leases/src/leases.h

@@ -26,7 +26,6 @@

 #include "lkowner.h"

 #include "locking.h"

 #include "upcall-utils.h"

-#include "tw.h"

 #include "timer-wheel.h"

 #include "leases-mem-types.h"

 #include "leases-messages.h"

diff --git a/xlators/performance/nl-cache/src/nl-cache.c b/xlators/performance/nl-cache/src/nl-cache.c

index a34b752..92dbccb 100644

--- a/xlators/performance/nl-cache/src/nl-cache.c

+++ b/xlators/performance/nl-cache/src/nl-cache.c

@@ -12,7 +12,6 @@

 #include "nl-cache.h"

 #include "statedump.h"

 #include "upcall-utils.h"

-#include "tw.h"

 static void

 nlc_dentry_op (call_frame_t *frame, xlator_t *this, gf_boolean_t multilink)

@@ -625,6 +624,8 @@ nlc_priv_dump (xlator_t *this)

 void

 fini (xlator_t *this)

 {

+        glusterfs_ctx_tw_put (this->ctx);

+

         return;

 }

@@ -700,17 +701,12 @@ init (xlator_t *this)

         INIT_LIST_HEAD (&conf->lru);

         time (&conf->last_child_down);

-        if (!glusterfs_global_timer_wheel (this)) {

-                gf_msg_debug (this->name, 0, "Initing the global timer wheel");

-                ret = glusterfs_global_timer_wheel_init (this->ctx);

-                if (ret) {

-                        gf_msg (this->name, GF_LOG_ERROR, 0,

-                                NLC_MSG_NO_TIMER_WHEEL,

-                                "Initing the global timer wheel failed");

-                                goto out;

-                }

+        conf->timer_wheel = glusterfs_ctx_tw_get (this->ctx);

+        if (!conf->timer_wheel) {

+                gf_msg (this->name, GF_LOG_ERROR, 0, NLC_MSG_NO_TIMER_WHEEL,

+                        "Initing the global timer wheel failed");

+                        goto out;

         }

-        conf->timer_wheel = glusterfs_global_timer_wheel (this);

         this->private = conf;

-- 

1.8.3.1