From 5d67d0a79a0986b3b547a80d6ebc3db5d70a7b77 Mon Sep 17 00:00:00 2001 From: Dan Lambright Date: Thu, 11 Jun 2015 14:45:52 -0400 Subject: [PATCH 56/57] tier/dht: Fixing non atomic promotion/demotion w.r.t to frequency period This is a backport of 11110 > This fixes the ping-pong issue i.e files getting demoted immediately > after promition, caused by off-sync promotion/demotion processes. > The solution is do promotion/demotion refering to the system time. > To have the fix working all the file serving nodes should have > thier system time synchronized with each other either manually or > using a NTP Server. > NOTE: The ping-pong issue can re-appear even with this fix, if the admin > have different promotion freq period and demotion freq period, but this > would be under the control of the admin. > Change-Id: I1b33a5881d0cac143662ddb48e5b7b653aeb1271 > BUG: 1218717 > Signed-off-by: Joseph Fernandes > Reviewed-on: http://review.gluster.org/11110 > Reviewed-by: Dan Lambright Change-Id: I27aa58017b75e5ba8977967176802bfb52ead656 BUG: 1229268 Signed-off-by: Dan Lambright Reviewed-on: https://code.engineering.redhat.com/gerrit/50581 Reviewed-by: Joseph Fernandes Tested-by: Joseph Fernandes Reviewed-by: Shyam Ranganathan --- xlators/cluster/dht/src/tier.c | 100 +++++++++++++++++++++++++-------------- 1 files changed, 64 insertions(+), 36 deletions(-) diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c index cef4f5c..7b4890c 100644 --- a/xlators/cluster/dht/src/tier.c +++ b/xlators/cluster/dht/src/tier.c @@ -734,7 +734,7 @@ tier_get_bricklist (xlator_t *xl, dict_t *bricklist) if (!db_path) { gf_msg ("tier", GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_STATUS, - "Failed to allocate memory for bricklist"); + "Faile. to allocate memory for bricklist"); goto out; } @@ -763,9 +763,7 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag) dict_t *bricklist_cold = NULL; dict_t *bricklist_hot = NULL; dht_conf_t *conf = NULL; - int tick = 0; - int next_demote = 0; - int next_promote = 0; + gfdb_time_t current_time; int freq_promote = 0; int freq_demote = 0; promotion_args_t promotion_args = { 0 }; @@ -775,6 +773,8 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag) int ret = 0; pthread_t promote_thread; pthread_t demote_thread; + gf_boolean_t is_promotion_triggered = _gf_false; + gf_boolean_t is_demotion_triggered = _gf_false; conf = this->private; @@ -789,16 +789,9 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag) tier_get_bricklist (conf->subvolumes[0], bricklist_cold); tier_get_bricklist (conf->subvolumes[1], bricklist_hot); - freq_promote = defrag->tier_promote_frequency; - freq_demote = defrag->tier_demote_frequency; - - next_promote = defrag->tier_promote_frequency % TIMER_SECS; - next_demote = defrag->tier_demote_frequency % TIMER_SECS; - - gf_msg (this->name, GF_LOG_INFO, 0, - DHT_MSG_LOG_TIER_STATUS, "Begin run tier promote %d demote %d", - next_promote, next_demote); + DHT_MSG_LOG_TIER_STATUS, "Begin run tier promote %d" + " demote %d", freq_promote, freq_demote); defrag->defrag_status = GF_DEFRAG_STATUS_STARTED; @@ -806,9 +799,6 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag) sleep(1); - ret_promotion = -1; - ret_demotion = -1; - if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) { ret = 1; gf_msg (this->name, GF_LOG_ERROR, 0, @@ -820,7 +810,8 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag) if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) { ret = 0; - defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE; + defrag->defrag_status = + GF_DEFRAG_STATUS_COMPLETE; gf_msg (this->name, GF_LOG_DEBUG, 0, DHT_MSG_LOG_TIER_ERROR, "defrag->defrag_cmd == " @@ -828,43 +819,75 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag) goto out; } - tick = (tick + 1) % TIMER_SECS; - if ((next_demote != tick) && (next_promote != tick)) + freq_promote = defrag->tier_promote_frequency; + freq_demote = defrag->tier_demote_frequency; + + /* To have proper synchronization amongst all + * brick holding nodes, so that promotion and demotions + * start atomicly w.r.t promotion/demotion frequency + * period, all nodes should have thier system time + * in-sync with each other either manually set or + * using a NTP server*/ + ret = gettimeofday (¤t_time, NULL); + if (ret == -1) { + gf_log (this->name, GF_LOG_ERROR, + "Failed to get current time"); + goto out; + } + + is_demotion_triggered = ((current_time.tv_sec % + freq_demote) == 0) ? _gf_true : + _gf_false; + is_promotion_triggered = ((current_time.tv_sec % + freq_promote) == 0) ? _gf_true : + _gf_false; + + /* If no promotion and no demotion is + * scheduled/triggered skip a iteration */ + if (!is_promotion_triggered && !is_demotion_triggered) continue; - if (next_demote >= tick) { + ret_promotion = -1; + ret_demotion = -1; + + if (is_demotion_triggered) { + demotion_args.this = this; demotion_args.brick_list = bricklist_hot; demotion_args.defrag = defrag; demotion_args.freq_time = freq_demote; - ret_demotion = pthread_create (&demote_thread, NULL, - &tier_demote, &demotion_args); + ret_demotion = pthread_create (&demote_thread, + NULL, &tier_demote, + &demotion_args); + if (ret_demotion) { gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "Failed starting Demotion thread!"); + "Failed starting Demotion " + "thread!"); + } - freq_demote = defrag->tier_demote_frequency; - next_demote = (tick + freq_demote) % TIMER_SECS; } - if (next_promote >= tick) { + if (is_promotion_triggered) { promotion_args.this = this; promotion_args.brick_list = bricklist_cold; promotion_args.defrag = defrag; promotion_args.freq_time = freq_promote; - ret_promotion = pthread_create (&promote_thread, NULL, - &tier_promote, &promotion_args); + ret_promotion = pthread_create (&promote_thread, + NULL, &tier_promote, + &promotion_args); + if (ret_promotion) { gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "Failed starting Promotion thread!"); + "Failed starting Promotion " + "thread!"); + } - freq_promote = defrag->tier_promote_frequency; - next_promote = (tick + freq_promote) % TIMER_SECS; } - if (ret_demotion == 0) { + if (is_promotion_triggered && (ret_demotion == 0)) { pthread_join (demote_thread, NULL); if (demotion_args.return_value) { gf_msg (this->name, GF_LOG_ERROR, 0, @@ -874,7 +897,7 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag) ret_demotion = demotion_args.return_value; } - if (ret_promotion == 0) { + if (is_demotion_triggered && (ret_promotion == 0)) { pthread_join (promote_thread, NULL); if (promotion_args.return_value) { gf_msg (this->name, GF_LOG_ERROR, 0, @@ -884,10 +907,15 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag) ret_promotion = promotion_args.return_value; } - /*Collect previous and current cummulative status */ - ret = ret | ret_demotion | ret_promotion; + /* Collect previous and current cummulative status */ + /* If demotion was not triggered just pass 0 to ret */ + ret = (is_demotion_triggered) ? ret_demotion : 0; + /* If promotion was not triggered just pass 0 to ret */ + ret = ret | (is_promotion_triggered) ? + ret_promotion : 0; - /*reseting promotion and demotion arguments for next iteration*/ + /* reseting promotion and demotion arguments for + * next iteration*/ memset (&demotion_args, 0, sizeof(demotion_args_t)); memset (&promotion_args, 0, sizeof(promotion_args_t)); -- 1.7.1