From e68845ff7018e5d81d7979684b18e6eda449b088 Mon Sep 17 00:00:00 2001 From: Sanju Rakonde Date: Thu, 8 Nov 2018 18:50:18 +0530 Subject: [PATCH 471/493] glusterd: migrating profile commands to mgmt_v3 framework Current profile commands use the op_state machine framework. Porting it to use the mgmt_v3 framework. The following tests were performed on the patch: case 1: 1. On a 3 node cluster, created and started 3 volumes 2. Mounted all the three volumes and wrote some data 3. Started profile operation for all the volumes 4. Ran "gluster v status" from N1, "gluster v profile info" form N2, "gluster v profile info" from N3 simultaneously in a loop for around 10000 times 5. Didn't find any cores generated. case 2: 1. Repeat the steps 1,2 and 3 from case 1. 2. Ran "gluster v status" from N1, "gluster v profile info" form N2(terminal 1), "gluster v profile info" from N2(terminal 2) simultaneously in a loop. 3. No cores were generated. > fixes: bz#1654181 > Change-Id: I83044cf5aee3970ef94066c89fcc41783ed468a6 > Signed-off-by: Sanju Rakonde upstream patch: https://review.gluster.org/#/c/glusterfs/+/21736/ Change-Id: I83044cf5aee3970ef94066c89fcc41783ed468a6 BUG: 1639476 Signed-off-by: Sanju Rakonde Reviewed-on: https://code.engineering.redhat.com/gerrit/158631 Tested-by: RHGS Build Bot Reviewed-by: Atin Mukherjee --- libglusterfs/src/globals.h | 2 + xlators/mgmt/glusterd/src/glusterd-handler.c | 18 +- xlators/mgmt/glusterd/src/glusterd-mgmt.c | 240 +++++++++++++++++++++++++-- xlators/mgmt/glusterd/src/glusterd-mgmt.h | 6 +- xlators/mgmt/glusterd/src/glusterd-op-sm.c | 4 +- xlators/mgmt/glusterd/src/glusterd-op-sm.h | 6 + 6 files changed, 252 insertions(+), 24 deletions(-) diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h index 1bede2e..d2b0964 100644 --- a/libglusterfs/src/globals.h +++ b/libglusterfs/src/globals.h @@ -111,6 +111,8 @@ #define GD_OP_VERSION_3_13_3 31303 /* Op-version for GlusterFS 3.13.3 */ +#define GD_OP_VERSION_6_0 60000 /* Op-version for GlusterFS 6.0 */ + /* Downstream only change */ #define GD_OP_VERSION_3_11_2 31102 /* Op-version for RHGS 3.3.1-async */ #define GD_OP_VERSION_3_13_3 31303 /* Op-version for RHGS-3.4-Batch Update-1*/ diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index 7486f51..90eaa95 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -28,6 +28,7 @@ #include "glusterd-sm.h" #include "glusterd-op-sm.h" #include "glusterd-utils.h" +#include "glusterd-mgmt.h" #include "glusterd-server-quorum.h" #include "glusterd-store.h" #include "glusterd-locks.h" @@ -3065,10 +3066,13 @@ __glusterd_handle_cli_profile_volume (rpcsvc_request_t *req) int32_t op = 0; char err_str[2048] = {0,}; xlator_t *this = NULL; + glusterd_conf_t *conf = NULL; GF_ASSERT (req); this = THIS; GF_ASSERT (this); + conf = this->private; + GF_VALIDATE_OR_GOTO(this->name, conf, out); ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req); if (ret < 0) { @@ -3109,12 +3113,18 @@ __glusterd_handle_cli_profile_volume (rpcsvc_request_t *req) goto out; } - ret = glusterd_op_begin (req, cli_op, dict, err_str, sizeof (err_str)); + if (conf->op_version < GD_OP_VERSION_6_0) { + gf_msg_debug(this->name, 0, "The cluster is operating at " + "version less than %d. Falling back to op-sm " + "framework.", GD_OP_VERSION_6_0); + ret = glusterd_op_begin(req, cli_op, dict, err_str, sizeof(err_str)); + glusterd_friend_sm(); + glusterd_op_sm(); + } else { + ret = glusterd_mgmt_v3_initiate_profile_phases(req, cli_op, dict); + } out: - glusterd_friend_sm (); - glusterd_op_sm (); - free (cli_req.dict.dict_val); if (ret) { diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c index d7da3c1..751d6e4 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mgmt.c +++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c @@ -19,6 +19,7 @@ #include "glusterd-locks.h" #include "glusterd-mgmt.h" #include "glusterd-op-sm.h" +#include "glusterd-server-quorum.h" #include "glusterd-volgen.h" #include "glusterd-store.h" #include "glusterd-snapshot-utils.h" @@ -213,6 +214,16 @@ gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict, } break; + case GD_OP_PROFILE_VOLUME: + ret = glusterd_op_stage_stats_volume(dict, op_errstr); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, 0, + GD_MSG_PRE_VALIDATION_FAIL, + "prevalidation failed for profile operation."); + goto out; + } + break; + case GD_OP_MAX_OPVERSION: ret = 0; break; @@ -252,6 +263,16 @@ gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict, } break; } + case GD_OP_PROFILE_VOLUME: + { + ret = gd_brick_op_phase(op, rsp_dict, dict, op_errstr); + if (ret) { + gf_log(this->name, GF_LOG_WARNING, "%s brickop failed", + gd_op_list[op]); + goto out; + } + break; + } default: break; } @@ -406,6 +427,17 @@ gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict, break; } + case GD_OP_PROFILE_VOLUME: + { + ret = glusterd_op_stats_volume(dict, op_errstr, rsp_dict); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, + GD_MSG_COMMIT_OP_FAIL, "commit failed " + "volume profile operation."); + goto out; + } + break; + } default: break; @@ -847,6 +879,7 @@ glusterd_pre_validate_aggr_rsp_dict (glusterd_op_t op, case GD_OP_DETACH_TIER_STATUS: case GD_OP_TIER_START_STOP: case GD_OP_REMOVE_TIER_BRICK: + case GD_OP_PROFILE_VOLUME: break; case GD_OP_MAX_OPVERSION: break; @@ -1039,6 +1072,16 @@ glusterd_mgmt_v3_pre_validate (glusterd_op_t op, dict_t *req_dict, goto out; } + if (op == GD_OP_PROFILE_VOLUME) { + ret = glusterd_validate_quorum(this, op, req_dict, op_errstr); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_SERVER_QUORUM_NOT_MET, "Server quorum " + "not met. Rejecting operation."); + goto out; + } + } + /* Pre Validation on local node */ ret = gd_mgmt_v3_pre_validate_fn (op, req_dict, op_errstr, rsp_dict, op_errno); @@ -1157,6 +1200,7 @@ glusterd_mgmt_v3_build_payload (dict_t **req, char **op_errstr, dict_t *dict, case GD_OP_REPLACE_BRICK: case GD_OP_RESET_BRICK: case GD_OP_ADD_TIER_BRICK: + case GD_OP_PROFILE_VOLUME: { ret = dict_get_str (dict, "volname", &volname); if (ret) { @@ -1309,12 +1353,11 @@ out: } int -glusterd_mgmt_v3_brick_op (glusterd_op_t op, dict_t *req_dict, char **op_errstr, - uint32_t txn_generation) +glusterd_mgmt_v3_brick_op (glusterd_op_t op, dict_t *rsp_dict, dict_t *req_dict, + char **op_errstr, uint32_t txn_generation) { int32_t ret = -1; int32_t peer_cnt = 0; - dict_t *rsp_dict = NULL; glusterd_peerinfo_t *peerinfo = NULL; struct syncargs args = {0}; uuid_t peer_uuid = {0}; @@ -1329,14 +1372,6 @@ glusterd_mgmt_v3_brick_op (glusterd_op_t op, dict_t *req_dict, char **op_errstr, GF_ASSERT (req_dict); GF_ASSERT (op_errstr); - rsp_dict = dict_new (); - if (!rsp_dict) { - gf_msg (this->name, GF_LOG_ERROR, 0, - GD_MSG_DICT_CREATE_FAIL, - "Failed to create response dictionary"); - goto out; - } - /* Perform brick op on local node */ ret = gd_mgmt_v3_brick_op_fn (op, req_dict, op_errstr, rsp_dict); @@ -1361,11 +1396,8 @@ glusterd_mgmt_v3_brick_op (glusterd_op_t op, dict_t *req_dict, char **op_errstr, goto out; } - dict_unref (rsp_dict); - rsp_dict = NULL; - /* Sending brick op req to other nodes in the cluster */ - gd_syncargs_init (&args, NULL); + gd_syncargs_init (&args, rsp_dict); synctask_barrier_init((&args)); peer_cnt = 0; @@ -2108,6 +2140,180 @@ out: } int32_t +glusterd_mgmt_v3_initiate_profile_phases (rpcsvc_request_t *req, + glusterd_op_t op, dict_t *dict) +{ + int32_t ret = -1; + int32_t op_ret = -1; + dict_t *req_dict = NULL; + dict_t *tmp_dict = NULL; + glusterd_conf_t *conf = NULL; + char *op_errstr = NULL; + xlator_t *this = NULL; + gf_boolean_t is_acquired = _gf_false; + uuid_t *originator_uuid = NULL; + uint32_t txn_generation = 0; + uint32_t op_errno = 0; + + this = THIS; + GF_ASSERT (this); + GF_ASSERT (req); + GF_ASSERT (dict); + conf = this->private; + GF_ASSERT (conf); + + /* Save the peer list generation */ + txn_generation = conf->generation; + cmm_smp_rmb (); + /* This read memory barrier makes sure that this assignment happens here + * only and is not reordered and optimized by either the compiler or the + * processor. + */ + + /* Save the MY_UUID as the originator_uuid. This originator_uuid + * will be used by is_origin_glusterd() to determine if a node + * is the originator node for a command. */ + originator_uuid = GF_CALLOC (1, sizeof(uuid_t), + gf_common_mt_uuid_t); + if (!originator_uuid) { + ret = -1; + goto out; + } + + gf_uuid_copy (*originator_uuid, MY_UUID); + ret = dict_set_bin (dict, "originator_uuid", + originator_uuid, sizeof (uuid_t)); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_SET_FAILED, + "Failed to set originator_uuid."); + GF_FREE (originator_uuid); + goto out; + } + + /* Marking the operation as complete synctasked */ + ret = dict_set_int32 (dict, "is_synctasked", _gf_true); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_SET_FAILED, + "Failed to set synctasked flag."); + goto out; + } + + /* Use a copy at local unlock as cli response will be sent before + * the unlock and the volname in the dict might be removed */ + tmp_dict = dict_new(); + if (!tmp_dict) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_CREATE_FAIL, "Unable to create dict"); + goto out; + } + dict_copy (dict, tmp_dict); + + /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */ + ret = glusterd_mgmt_v3_initiate_lockdown (op, dict, &op_errstr, + &op_errno, &is_acquired, + txn_generation); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_MGMTV3_LOCKDOWN_FAIL, + "mgmt_v3 lockdown failed."); + goto out; + } + + /* BUILD PAYLOAD */ + ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL, LOGSTR_BUILD_PAYLOAD, + gd_op_list[op]); + if (op_errstr == NULL) + gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD); + goto out; + } + + /* PRE-COMMIT VALIDATE PHASE */ + ret = glusterd_mgmt_v3_pre_validate (op, req_dict, &op_errstr, + &op_errno, txn_generation); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_PRE_VALIDATION_FAIL, "Pre Validation Failed"); + goto out; + } + + /* BRICK-OPS */ + ret = glusterd_mgmt_v3_brick_op(op, dict, req_dict, &op_errstr, + txn_generation); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, "Brick Op Failed"); + goto out; + } + + /* COMMIT OP PHASE */ + ret = glusterd_mgmt_v3_commit (op, dict, req_dict, &op_errstr, + &op_errno, txn_generation); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_COMMIT_OP_FAIL, "Commit Op Failed"); + goto out; + } + + /* POST-COMMIT VALIDATE PHASE */ + /* As of now, post_validate is not trying to cleanup any failed + commands. So as of now, I am sending 0 (op_ret as 0). + */ + ret = glusterd_mgmt_v3_post_validate (op, 0, dict, req_dict, &op_errstr, + txn_generation); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_POST_VALIDATION_FAIL, "Post Validation Failed"); + goto out; + } + + ret = 0; +out: + op_ret = ret; + /* UNLOCK PHASE FOR PEERS*/ + (void) glusterd_mgmt_v3_release_peer_locks (op, dict, op_ret, + &op_errstr, is_acquired, + txn_generation); + + /* LOCAL VOLUME(S) UNLOCK */ + if (is_acquired) { + /* Trying to release multiple mgmt_v3 locks */ + ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_MGMTV3_UNLOCK_FAIL, + "Failed to release mgmt_v3 locks on localhost"); + op_ret = ret; + } + } + + if (op_ret && (op_errno == 0)) + op_errno = EG_INTRNL; + + if (op != GD_OP_MAX_OPVERSION) { + /* SEND CLI RESPONSE */ + glusterd_op_send_cli_response (op, op_ret, op_errno, req, + dict, op_errstr); + } + + if (req_dict) + dict_unref (req_dict); + + if (tmp_dict) + dict_unref (tmp_dict); + + if (op_errstr) { + GF_FREE (op_errstr); + op_errstr = NULL; + } + + return 0; +} + +int32_t glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op, dict_t *dict) { @@ -2465,7 +2671,7 @@ glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op, goto out; } - ret = glusterd_mgmt_v3_brick_op (op, req_dict, &op_errstr, + ret = glusterd_mgmt_v3_brick_op (op, dict, req_dict, &op_errstr, txn_generation); if (ret) { gf_msg (this->name, GF_LOG_ERROR, 0, @@ -2526,7 +2732,7 @@ unbarrier: goto out; } - ret = glusterd_mgmt_v3_brick_op (op, req_dict, &op_errstr, + ret = glusterd_mgmt_v3_brick_op (op, dict, req_dict, &op_errstr, txn_generation); if (ret) { diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-mgmt.h index 2215f17..eff070d 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mgmt.h +++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.h @@ -37,7 +37,11 @@ glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op, dict_t *dict); int32_t -glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op, +glusterd_mgmt_v3_initiate_profile_phases(rpcsvc_request_t *req, + glusterd_op_t op, dict_t *dict); + +int32_t +glusterd_mgmt_v3_initiate_snap_phases(rpcsvc_request_t *req, glusterd_op_t op, dict_t *dict); int diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c index 52a3db0..9f76ab3 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c @@ -2032,7 +2032,7 @@ glusterd_op_stage_status_volume (dict_t *dict, char **op_errstr) return ret; } -static int +int glusterd_op_stage_stats_volume (dict_t *dict, char **op_errstr) { int ret = -1; @@ -3322,7 +3322,7 @@ glusterd_remove_profile_volume_options (glusterd_volinfo_t *volinfo) dict_del (volinfo->dict, fd_stats_key); } -static int +int glusterd_op_stats_volume (dict_t *dict, char **op_errstr, dict_t *rsp_dict) { diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h index f2aee9c..e64d368 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h @@ -312,4 +312,10 @@ glusterd_dict_set_volid (dict_t *dict, char *volname, char **op_errstr); int32_t glusterd_tier_op (xlator_t *this, void *data); + +int +glusterd_op_stats_volume (dict_t *dict, char **op_errstr, dict_t *rsp_dict); + +int +glusterd_op_stage_stats_volume (dict_t *dict, char **op_errstr); #endif -- 1.8.3.1