From 5d67d0a79a0986b3b547a80d6ebc3db5d70a7b77 Mon Sep 17 00:00:00 2001

From: Dan Lambright <dlambrig@redhat.com>

Date: Thu, 11 Jun 2015 14:45:52 -0400

Subject: [PATCH 56/57] tier/dht: Fixing non atomic promotion/demotion w.r.t to frequency period

This is a backport of 11110

> This fixes the ping-pong issue i.e files getting demoted immediately

> after promition, caused by off-sync promotion/demotion processes.

> The solution is do promotion/demotion refering to the system time.

> To have the fix working all the file serving nodes should have

> thier system time synchronized with each other either manually or

> using a NTP Server.

> NOTE: The ping-pong issue can re-appear even with this fix, if the admin

> have different promotion freq period and demotion freq period, but this

> would be under the control of the admin.

> Change-Id: I1b33a5881d0cac143662ddb48e5b7b653aeb1271

> BUG: 1218717

> Signed-off-by: Joseph Fernandes <josferna@redhat.com>

> Reviewed-on: http://review.gluster.org/11110

> Reviewed-by: Dan Lambright <dlambrig@redhat.com>

Change-Id: I27aa58017b75e5ba8977967176802bfb52ead656

BUG: 1229268

Signed-off-by: Dan Lambright <dlambrig@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/50581

Reviewed-by: Joseph Fernandes <josferna@redhat.com>

Tested-by: Joseph Fernandes <josferna@redhat.com>

Reviewed-by: Shyam Ranganathan <srangana@redhat.com>

---

 xlators/cluster/dht/src/tier.c |  100 +++++++++++++++++++++++++--------------

 1 files changed, 64 insertions(+), 36 deletions(-)

diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c

index cef4f5c..7b4890c 100644

--- a/xlators/cluster/dht/src/tier.c

+++ b/xlators/cluster/dht/src/tier.c

@@ -734,7 +734,7 @@ tier_get_bricklist (xlator_t *xl, dict_t *bricklist)

                         if (!db_path) {

                                 gf_msg ("tier", GF_LOG_ERROR, 0,

                                         DHT_MSG_LOG_TIER_STATUS,

-                                        "Failed to allocate memory for bricklist");

+                                        "Faile. to allocate memory for bricklist");

                                 goto out;

                         }

@@ -763,9 +763,7 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)

         dict_t       *bricklist_cold = NULL;

         dict_t       *bricklist_hot = NULL;

         dht_conf_t   *conf     = NULL;

-        int tick = 0;

-        int next_demote = 0;

-        int next_promote = 0;

+        gfdb_time_t  current_time;

         int freq_promote = 0;

         int freq_demote = 0;

         promotion_args_t promotion_args = { 0 };

@@ -775,6 +773,8 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)

         int ret = 0;

         pthread_t promote_thread;

         pthread_t demote_thread;

+        gf_boolean_t  is_promotion_triggered = _gf_false;

+        gf_boolean_t  is_demotion_triggered = _gf_false;

         conf   = this->private;

@@ -789,16 +789,9 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)

         tier_get_bricklist (conf->subvolumes[0], bricklist_cold);

         tier_get_bricklist (conf->subvolumes[1], bricklist_hot);

-        freq_promote = defrag->tier_promote_frequency;

-        freq_demote  = defrag->tier_demote_frequency;

-

-        next_promote = defrag->tier_promote_frequency % TIMER_SECS;

-        next_demote  = defrag->tier_demote_frequency % TIMER_SECS;

-

-

         gf_msg (this->name, GF_LOG_INFO, 0,

-                DHT_MSG_LOG_TIER_STATUS, "Begin run tier promote %d demote %d",

-                next_promote, next_demote);

+                DHT_MSG_LOG_TIER_STATUS, "Begin run tier promote %d"

+                        " demote %d", freq_promote, freq_demote);

         defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;

@@ -806,9 +799,6 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)

                 sleep(1);

-                ret_promotion = -1;

-                ret_demotion = -1;

-

                 if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {

                         ret = 1;

                         gf_msg (this->name, GF_LOG_ERROR, 0,

@@ -820,7 +810,8 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)

                 if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) {

                         ret = 0;

-                        defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;

+                        defrag->defrag_status =

+                                        GF_DEFRAG_STATUS_COMPLETE;

                         gf_msg (this->name, GF_LOG_DEBUG, 0,

                                 DHT_MSG_LOG_TIER_ERROR,

                                 "defrag->defrag_cmd == "

@@ -828,43 +819,75 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)

                         goto out;

                 }

-                tick = (tick + 1) % TIMER_SECS;

-                if ((next_demote != tick) && (next_promote != tick))

+                freq_promote = defrag->tier_promote_frequency;

+                freq_demote  = defrag->tier_demote_frequency;

+

+                /* To have proper synchronization amongst all

+                 * brick holding nodes, so that promotion and demotions

+                 * start atomicly w.r.t promotion/demotion frequency

+                 * period, all nodes should have thier system time

+                 * in-sync with each other either manually set or

+                 * using a NTP server*/

+                ret = gettimeofday (&current_time, NULL);

+                if (ret == -1) {

+                        gf_log (this->name, GF_LOG_ERROR,

+                                "Failed to get current time");

+                        goto out;

+                }

+

+                is_demotion_triggered = ((current_time.tv_sec %

+                                        freq_demote) == 0) ? _gf_true :

+                                        _gf_false;

+                is_promotion_triggered = ((current_time.tv_sec %

+                                        freq_promote) == 0) ? _gf_true :

+                                        _gf_false;

+

+                /* If no promotion and no demotion is

+                 * scheduled/triggered skip a iteration */

+                if (!is_promotion_triggered && !is_demotion_triggered)

                         continue;

-                if (next_demote >= tick) {

+                ret_promotion = -1;

+                ret_demotion = -1;

+

+                if (is_demotion_triggered) {

+

                         demotion_args.this = this;

                         demotion_args.brick_list = bricklist_hot;

                         demotion_args.defrag = defrag;

                         demotion_args.freq_time = freq_demote;

-                        ret_demotion = pthread_create (&demote_thread, NULL,

-                                        &tier_demote, &demotion_args);

+                        ret_demotion = pthread_create (&demote_thread,

+                                                NULL, &tier_demote,

+                                                &demotion_args);

+

                         if (ret_demotion) {

                                 gf_msg (this->name, GF_LOG_ERROR, 0,

                                         DHT_MSG_LOG_TIER_ERROR,

-                                        "Failed starting Demotion thread!");

+                                        "Failed starting Demotion "

+                                        "thread!");

+

                         }

-                        freq_demote = defrag->tier_demote_frequency;

-                        next_demote = (tick + freq_demote) % TIMER_SECS;

                 }

-                if (next_promote >= tick) {

+                if (is_promotion_triggered) {

                         promotion_args.this = this;

                         promotion_args.brick_list = bricklist_cold;

                         promotion_args.defrag = defrag;

                         promotion_args.freq_time = freq_promote;

-                        ret_promotion = pthread_create (&promote_thread, NULL,

-                                                &tier_promote, &promotion_args);

+                        ret_promotion = pthread_create (&promote_thread,

+                                                NULL, &tier_promote,

+                                                &promotion_args);

+

                         if (ret_promotion) {

                                 gf_msg (this->name, GF_LOG_ERROR, 0,

                                         DHT_MSG_LOG_TIER_ERROR,

-                                        "Failed starting Promotion thread!");

+                                        "Failed starting Promotion "

+                                        "thread!");

+

                         }

-                        freq_promote = defrag->tier_promote_frequency;

-                        next_promote = (tick + freq_promote) % TIMER_SECS;

                 }

-                if (ret_demotion == 0) {

+                if (is_promotion_triggered && (ret_demotion == 0)) {

                         pthread_join (demote_thread, NULL);

                         if (demotion_args.return_value) {

                                 gf_msg (this->name, GF_LOG_ERROR, 0,

@@ -874,7 +897,7 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)

                         ret_demotion = demotion_args.return_value;

                 }

-                if (ret_promotion == 0) {

+                if (is_demotion_triggered && (ret_promotion == 0)) {

                         pthread_join (promote_thread, NULL);

                         if (promotion_args.return_value) {

                                 gf_msg (this->name, GF_LOG_ERROR, 0,

@@ -884,10 +907,15 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)

                         ret_promotion = promotion_args.return_value;

                 }

-                /*Collect previous and current cummulative status */

-                ret = ret | ret_demotion | ret_promotion;

+                /* Collect previous and current cummulative status */

+                /* If demotion was not triggered just pass 0 to ret */

+                ret = (is_demotion_triggered) ? ret_demotion : 0;

+                /* If promotion was not triggered just pass 0 to ret */

+                ret = ret | (is_promotion_triggered) ?

+                                ret_promotion : 0;

-                /*reseting promotion and demotion arguments for next iteration*/

+                /* reseting promotion and demotion arguments for

+                 * next iteration*/

                 memset (&demotion_args, 0, sizeof(demotion_args_t));

                 memset (&promotion_args, 0, sizeof(promotion_args_t));

-- 

1.7.1