From 68b385605f52503f8c80ebf0aa15fd278b5db6d0 Mon Sep 17 00:00:00 2001
From: N Balachandran <nbalacha@redhat.com>
Date: Tue, 17 Jan 2017 15:43:47 +0530
Subject: [PATCH 314/361] dht/rebalance Estimate time to complete rebalance
The estimates will be logged to the rebalance log on running
gluster v rebalance <vol> status
mainline:
> BUG: 1396004
> Reviewed-on: http://review.gluster.org/15893
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
(cherry picked from commit 310405d689760fc1abe3075c4bcf6cf7351e60bc)
BUG: 1380598
Change-Id: I9d51b139cd4c8dfde1ff2c2050720ae606c13fc6
Signed-off-by: N Balachandran <nbalacha@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/101294
Tested-by: Milind Changire <mchangir@redhat.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
---
cli/src/cli-rpc-ops.c | 28 +++-
xlators/cluster/dht/src/dht-rebalance.c | 102 +++++++++++-
xlators/mgmt/glusterd/src/glusterd-utils.c | 251 ++++++++++++++++++++++++++++-
xlators/mgmt/glusterd/src/glusterd.h | 1 +
4 files changed, 376 insertions(+), 6 deletions(-)
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 5a0cfdf..339d91b 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -1611,6 +1611,9 @@ gf_cli_print_rebalance_status (dict_t *dict, enum gf_task_types task_type,
int sec = 0;
gf_boolean_t down = _gf_false;
gf_boolean_t fix_layout = _gf_false;
+ uint64_t max_time = 0;
+ uint64_t time_left = 0;
+
ret = dict_get_int32 (dict, "count", &count);
if (ret) {
@@ -1663,6 +1666,7 @@ gf_cli_print_rebalance_status (dict_t *dict, enum gf_task_types task_type,
skipped = 0;
status_str = NULL;
elapsed = 0;
+ time_left = 0;
/* Check if status is NOT_STARTED, and continue early */
memset (key, 0, 256);
@@ -1740,6 +1744,15 @@ gf_cli_print_rebalance_status (dict_t *dict, enum gf_task_types task_type,
if (ret)
gf_log ("cli", GF_LOG_TRACE, "failed to get run-time");
+ memset (key, 0, 256);
+ snprintf (key, 256, "time-left-%d", i);
+ ret = dict_get_uint64 (dict, key, &time_left);
+ if (ret)
+ gf_log ("cli", GF_LOG_TRACE,
+ "failed to get time left");
+ if (time_left > max_time)
+ max_time = time_left;
+
/* Check for array bound */
if (status_rcd >= GF_DEFRAG_STATUS_MAX)
status_rcd = GF_DEFRAG_STATUS_MAX;
@@ -1757,15 +1770,15 @@ gf_cli_print_rebalance_status (dict_t *dict, enum gf_task_types task_type,
if (size_str) {
cli_out ("%40s %16"PRIu64 " %13s" " %13"PRIu64
" %13" PRIu64" %13"PRIu64 " %20s "
- "%8d:%d:%d", node_name, files,
+ "%8d:%02d:%02d", node_name, files,
size_str, lookup, failures, skipped,
status_str, hrs, min, sec);
} else {
cli_out ("%40s %16"PRIu64 " %13"PRIu64 " %13"
PRIu64 " %13"PRIu64" %13"PRIu64 " %20s"
- " %8d:%d:%d", node_name, files, size,
- lookup, failures, skipped, status_str,
- hrs, min, sec);
+ " %8d:%02d:%02d", node_name, files,
+ size, lookup, failures, skipped,
+ status_str, hrs, min, sec);
}
}
GF_FREE(size_str);
@@ -1775,6 +1788,13 @@ gf_cli_print_rebalance_status (dict_t *dict, enum gf_task_types task_type,
" Please check the nodes that are down using \'gluster"
" peer status\' and start the glusterd on those nodes,"
" else tier detach commit might fail!");
+ if (max_time) {
+ hrs = max_time / 3600;
+ min = ((int) max_time % 3600) / 60;
+ sec = ((int) max_time % 3600) % 60;
+ cli_out ("Estimated time left for rebalance to complete :"
+ " %8d:%02d:%02d", hrs, min, sec);
+ }
out:
return ret;
}
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 39a7bb6..76778de 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -46,6 +46,9 @@
} \
} \
+uint64_t g_totalfiles = 0;
+
+
void
gf_defrag_free_container (struct dht_container *container)
{
@@ -3701,6 +3704,58 @@ gf_tier_wait_fix_lookup (gf_defrag_info_t *defrag) {
/******************Tier background Fix layout functions END********************/
+uint64_t gf_defrag_subvol_file_cnt (xlator_t *this, loc_t *root_loc)
+{
+ int ret = -1;
+ struct statvfs buf = {0,};
+
+ if (!this)
+ return 0;
+
+ ret = syncop_statfs (this, root_loc, &buf, NULL, NULL);
+ if (ret) {
+ /* Aargh! */
+ return 0;
+ }
+ return (buf.f_files - buf.f_ffree);
+}
+
+
+int gf_defrag_total_file_cnt (xlator_t *this, loc_t *root_loc)
+{
+ dht_conf_t *conf = NULL;
+ int ret = -1;
+ int i = 0;
+ uint64_t num_files = 0;
+
+
+ conf = this->private;
+ if (!conf) {
+ return ret;
+ }
+
+ for (i = 0 ; i < conf->local_subvols_cnt; i++) {
+ num_files = gf_defrag_subvol_file_cnt (conf->local_subvols[i],
+ root_loc);
+ g_totalfiles += num_files;
+ gf_msg (this->name, GF_LOG_INFO, 0, 0, "local subvol: %s,"
+ "cnt = %"PRIu64, conf->local_subvols[i]->name,
+ num_files);
+ }
+
+ /* FIXFIXFIX: halve the number of files to negate .glusterfs contents
+ We need a better way to figure this out */
+
+ g_totalfiles = g_totalfiles/2;
+ if (g_totalfiles > 20000)
+ g_totalfiles += 10000;
+
+ gf_msg (this->name, GF_LOG_INFO, 0, 0,
+ "Total number of files = %"PRIu64, g_totalfiles);
+
+ return 0;
+}
+
int
gf_defrag_start_crawl (void *data)
@@ -3725,6 +3780,7 @@ gf_defrag_start_crawl (void *data)
pthread_t *tid = NULL;
gf_boolean_t is_tier_detach = _gf_false;
+
this = data;
if (!this)
goto exit;
@@ -3854,6 +3910,13 @@ gf_defrag_start_crawl (void *data)
"are %s", conf->local_subvols[i]->name);
}
+ ret = gf_defrag_total_file_cnt (this, &loc);
+ if (!ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, 0, "Failed to get "
+ "the total number of files. Unable to estimate "
+ "time to complete rebalance.");
+ }
+
/* Initialize global entry queue */
defrag->queue = GF_CALLOC (1, sizeof (struct dht_container),
gf_dht_mt_container_t);
@@ -4090,8 +4153,11 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
uint64_t skipped = 0;
uint64_t promoted = 0;
uint64_t demoted = 0;
- char *status = "";
+ char *status = "";
double elapsed = 0;
+ uint64_t time_left = 0;
+ uint64_t time_to_complete = 0;
+ double rate_lookedup = 0;
struct timeval end = {0,};
@@ -4114,6 +4180,34 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
elapsed = end.tv_sec - defrag->start_time.tv_sec;
+/*START */
+
+/* rate at which files looked up */
+
+
+ if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER)
+ && (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED)
+ && g_totalfiles) {
+
+ rate_lookedup = (defrag->num_files_lookedup)/elapsed;
+ if (defrag->num_files_lookedup > g_totalfiles)
+ g_totalfiles = defrag->num_files_lookedup + 10000;
+ time_to_complete = (g_totalfiles)/rate_lookedup;
+ time_left = time_to_complete - elapsed;
+
+ gf_log (THIS->name, GF_LOG_INFO,
+ "TIME: num_files_lookedup=%"PRIu64",elapsed time = %f,"
+ "rate_lookedup=%f", defrag->num_files_lookedup, elapsed,
+ rate_lookedup);
+ gf_log (THIS->name, GF_LOG_INFO,
+ "TIME: Estimated total time to complete = %"PRIu64
+ " seconds", time_to_complete);
+
+ gf_log (THIS->name, GF_LOG_INFO,
+ "TIME: Seconds left = %"PRIu64" seconds", time_left);
+ }
+
+/*END */
if (!dict)
goto log;
@@ -4162,6 +4256,12 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
if (ret)
gf_log (THIS->name, GF_LOG_WARNING,
"failed to set skipped file count");
+
+ ret = dict_set_uint64 (dict, "time-left", time_left);
+ if (ret)
+ gf_log (THIS->name, GF_LOG_WARNING,
+ "failed to set time-left");
+
log:
switch (defrag->defrag_status) {
case GF_DEFRAG_STATUS_NOT_STARTED:
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index d849690..e976ba2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -7586,6 +7586,7 @@ glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
dict_t *rsp_dict)
{
int ret = 0;
+ int ret2 = 0;
uint64_t files = 0;
uint64_t size = 0;
uint64_t lookup = 0;
@@ -7596,6 +7597,7 @@ glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
double run_time = 0;
uint64_t promoted = 0;
uint64_t demoted = 0;
+ uint64_t time_left = 0;
this = THIS;
@@ -7658,6 +7660,11 @@ glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
volinfo->rebal.skipped_files = skipped;
if (run_time)
volinfo->rebal.rebalance_time = run_time;
+ ret2 = dict_get_uint64 (rsp_dict, "time-left", &time_left);
+ if (ret2)
+ gf_msg_trace (this->name, 0,
+ "failed to get time left");
+
if (promoted)
volinfo->tier_info.promoted = promoted;
if (demoted)
@@ -9254,7 +9261,7 @@ glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
ret = dict_set_uint64 (ctx_dict, key, value);
if (ret) {
gf_msg_debug (THIS->name, 0,
- "failed to set lookuped file count");
+ "failed to set looked up file count");
}
}
@@ -9310,6 +9317,18 @@ glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
}
memset (key, 0, 256);
+ snprintf (key, 256, "time-left-%d", index);
+ ret = dict_get_uint64 (rsp_dict, key, &value);
+ if (!ret) {
+ memset (key, 0, 256);
+ snprintf (key, 256, "time-left-%d", current_index);
+ ret = dict_set_uint64 (ctx_dict, key, value);
+ if (ret) {
+ gf_msg_debug (THIS->name, 0,
+ "failed to set time-left");
+ }
+ }
+ memset (key, 0, 256);
snprintf (key, 256, "demoted-%d", index);
ret = dict_get_uint64 (rsp_dict, key, &value);
if (!ret) {
@@ -9341,6 +9360,228 @@ out:
}
int
+<<<<<<< 07a9e00a5702e76932142e9d9cdc2df601632b7a
+=======
+glusterd_volume_tier_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+ char key[256] = {0,};
+ char *node_uuid = NULL;
+ char *node_uuid_str = NULL;
+ char *volname = NULL;
+ dict_t *ctx_dict = NULL;
+ double elapsed_time = 0;
+ glusterd_volinfo_t *volinfo = NULL;
+ int ret = 0;
+ int32_t index = 0;
+ int32_t count = 0;
+ int32_t value32 = 0;
+ uint64_t value = 0;
+ xlator_t *this = NULL;
+ char *task_id_str = NULL;
+
+ this = THIS;
+ GF_VALIDATE_OR_GOTO (this->name, this, out);
+ GF_VALIDATE_OR_GOTO (this->name, rsp_dict, out);
+
+ if (aggr) {
+ ctx_dict = aggr;
+
+ } else {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_OPCTX_GET_FAIL,
+ "Operation Context is not present");
+ goto out;
+ }
+
+ if (!ctx_dict)
+ goto out;
+
+ ret = dict_get_str (ctx_dict, "volname", &volname);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_DICT_GET_FAILED,
+ "Unable to get volume name");
+ goto out;
+ }
+
+ ret = glusterd_volinfo_find (volname, &volinfo);
+
+ if (ret)
+ goto out;
+
+ ret = dict_get_int32 (rsp_dict, "count", &index);
+ if (ret)
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_DICT_GET_FAILED,
+ "failed to get index");
+
+ memset (key, 0, 256);
+ snprintf (key, 256, "node-uuid-%d", index);
+ ret = dict_get_str (rsp_dict, key, &node_uuid);
+ if (!ret) {
+ node_uuid_str = gf_strdup (node_uuid);
+
+ }
+ ret = dict_get_int32 (ctx_dict, "count", &count);
+ count++;
+ ret = dict_set_int32 (ctx_dict, "count", count);
+ if (ret)
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_DICT_SET_FAILED,
+ "Failed to set count");
+
+ memset (key, 0, 256);
+ snprintf (key, 256, "node-uuid-%d", count);
+ ret = dict_set_dynstr (ctx_dict, key, node_uuid_str);
+ if (ret) {
+ gf_msg_debug (this->name, 0,
+ "failed to set node-uuid");
+ }
+
+ snprintf (key, 256, "files-%d", index);
+ ret = dict_get_uint64 (rsp_dict, key, &value);
+ if (!ret) {
+ memset (key, 0, 256);
+ snprintf (key, 256, "files-%d", count);
+ ret = dict_set_uint64 (ctx_dict, key, value);
+ if (ret) {
+ gf_msg_debug (this->name, 0,
+ "failed to set the file count");
+ }
+ }
+
+ memset (key, 0, 256);
+ snprintf (key, 256, "size-%d", index);
+ ret = dict_get_uint64 (rsp_dict, key, &value);
+ if (!ret) {
+ memset (key, 0, 256);
+ snprintf (key, 256, "size-%d", count);
+ ret = dict_set_uint64 (ctx_dict, key, value);
+ if (ret) {
+ gf_msg_debug (this->name, 0,
+ "failed to set the size of migration");
+ }
+ }
+
+ memset (key, 0, 256);
+ snprintf (key, 256, "lookups-%d", index);
+ ret = dict_get_uint64 (rsp_dict, key, &value);
+ if (!ret) {
+ memset (key, 0, 256);
+ snprintf (key, 256, "lookups-%d", count);
+ ret = dict_set_uint64 (ctx_dict, key, value);
+ if (ret) {
+ gf_msg_debug (this->name, 0,
+ "failed to set looked up file count");
+ }
+ }
+
+ memset (key, 0, 256);
+ snprintf (key, 256, "status-%d", index);
+ ret = dict_get_int32 (rsp_dict, key, &value32);
+ if (!ret) {
+ memset (key, 0, 256);
+ snprintf (key, 256, "status-%d", count);
+ ret = dict_set_int32 (ctx_dict, key, value32);
+ if (ret) {
+ gf_msg_debug (this->name, 0,
+ "failed to set status");
+ }
+ }
+
+ memset (key, 0, 256);
+ snprintf (key, 256, "failures-%d", index);
+ ret = dict_get_uint64 (rsp_dict, key, &value);
+ if (!ret) {
+ memset (key, 0, 256);
+ snprintf (key, 256, "failures-%d", count);
+ ret = dict_set_uint64 (ctx_dict, key, value);
+ if (ret) {
+ gf_msg_debug (this->name, 0,
+ "failed to set failure count");
+ }
+ }
+
+ memset (key, 0, 256);
+ snprintf (key, 256, "skipped-%d", index);
+ ret = dict_get_uint64 (rsp_dict, key, &value);
+ if (!ret) {
+ memset (key, 0, 256);
+ snprintf (key, 256, "skipped-%d", count);
+ ret = dict_set_uint64 (ctx_dict, key, value);
+ if (ret) {
+ gf_msg_debug (this->name, 0,
+ "failed to set skipped count");
+ }
+ }
+ memset (key, 0, 256);
+ snprintf (key, 256, "run-time-%d", index);
+ ret = dict_get_double (rsp_dict, key, &elapsed_time);
+ if (!ret) {
+ memset (key, 0, 256);
+ snprintf (key, 256, "run-time-%d", count);
+ ret = dict_set_double (ctx_dict, key, elapsed_time);
+ if (ret) {
+ gf_msg_debug (this->name, 0,
+ "failed to set run-time");
+ }
+ }
+
+ memset (key, 0, 256);
+ snprintf (key, 256, "demoted-%d", index);
+ ret = dict_get_uint64 (rsp_dict, key, &value);
+ if (!ret) {
+ memset (key, 0, 256);
+ snprintf (key, 256, "demoted-%d", count);
+ ret = dict_set_uint64 (ctx_dict, key, value);
+ if (ret) {
+ gf_msg_debug (this->name, 0,
+ "failed to set demoted count");
+ }
+ }
+ memset (key, 0, 256);
+ snprintf (key, 256, "promoted-%d", index);
+ ret = dict_get_uint64 (rsp_dict, key, &value);
+ if (!ret) {
+ memset (key, 0, 256);
+ snprintf (key, 256, "promoted-%d", count);
+ ret = dict_set_uint64 (ctx_dict, key, value);
+ if (ret) {
+ gf_msg_debug (this->name, 0,
+ "failed to set promoted count");
+ }
+ }
+
+ memset (key, 0, 256);
+ snprintf (key, 256, "time-left-%d", index);
+ ret = dict_get_uint64 (rsp_dict, key, &value);
+ if (!ret) {
+ memset (key, 0, 256);
+ snprintf (key, 256, "time-left-%d", count);
+ ret = dict_set_uint64 (ctx_dict, key, value);
+ if (ret) {
+ gf_msg_debug (THIS->name, 0,
+ "failed to set time-left");
+ }
+ }
+
+ ret = dict_get_str (rsp_dict, GF_REMOVE_BRICK_TID_KEY,
+ &task_id_str);
+ if (ret) {
+ gf_msg_debug (this->name, errno,
+ "Missing remove-brick-id");
+ } else
+ ret = dict_set_str (ctx_dict, GF_REMOVE_BRICK_TID_KEY,
+ task_id_str);
+
+ ret = 0;
+
+out:
+ return ret;
+}
+
+int
+>>>>>>> dht/rebalance Estimate time to complete rebalance
glusterd_sys_exec_output_rsp_dict (dict_t *dst, dict_t *src)
{
char output_name[PATH_MAX] = "";
@@ -9976,6 +10217,14 @@ glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
"failed to set run-time");
memset (key, 0 , 256);
+ snprintf (key, 256, "time-left-%d", i);
+ ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.time_left);
+ if (ret)
+ gf_msg (THIS->name, GF_LOG_ERROR, errno,
+ GD_MSG_DICT_SET_FAILED,
+ "failed to set time left");
+
+ memset (key, 0 , 256);
snprintf (key, 256, "promoted-%d", i);
ret = dict_set_uint64 (op_ctx, key, volinfo->tier_info.promoted);
if (ret)
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index a21b0a1..c84f019 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -308,6 +308,7 @@ struct glusterd_rebalance_ {
uint64_t rebalance_failures;
uuid_t rebalance_id;
double rebalance_time;
+ uint64_t time_left;
glusterd_op_t op;
dict_t *dict; /* Dict to store misc information
* like list of bricks being removed */
--
1.8.3.1