From e68845ff7018e5d81d7979684b18e6eda449b088 Mon Sep 17 00:00:00 2001
From: Sanju Rakonde <srakonde@redhat.com>
Date: Thu, 8 Nov 2018 18:50:18 +0530
Subject: [PATCH 471/493] glusterd: migrating profile commands to mgmt_v3
framework
Current profile commands use the op_state machine framework.
Porting it to use the mgmt_v3 framework.
The following tests were performed on the patch:
case 1:
1. On a 3 node cluster, created and started 3 volumes
2. Mounted all the three volumes and wrote some data
3. Started profile operation for all the volumes
4. Ran "gluster v status" from N1,
"gluster v profile <volname1> info" form N2,
"gluster v profile <volname2> info" from N3 simultaneously in a
loop for around 10000 times
5. Didn't find any cores generated.
case 2:
1. Repeat the steps 1,2 and 3 from case 1.
2. Ran "gluster v status" from N1,
"gluster v profile <volname1> info" form N2(terminal 1),
"gluster v profile <volname2> info" from N2(terminal 2)
simultaneously in a loop.
3. No cores were generated.
> fixes: bz#1654181
> Change-Id: I83044cf5aee3970ef94066c89fcc41783ed468a6
> Signed-off-by: Sanju Rakonde <srakonde@redhat.com>
upstream patch: https://review.gluster.org/#/c/glusterfs/+/21736/
Change-Id: I83044cf5aee3970ef94066c89fcc41783ed468a6
BUG: 1639476
Signed-off-by: Sanju Rakonde <srakonde@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/158631
Tested-by: RHGS Build Bot <nigelb@redhat.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
---
libglusterfs/src/globals.h | 2 +
xlators/mgmt/glusterd/src/glusterd-handler.c | 18 +-
xlators/mgmt/glusterd/src/glusterd-mgmt.c | 240 +++++++++++++++++++++++++--
xlators/mgmt/glusterd/src/glusterd-mgmt.h | 6 +-
xlators/mgmt/glusterd/src/glusterd-op-sm.c | 4 +-
xlators/mgmt/glusterd/src/glusterd-op-sm.h | 6 +
6 files changed, 252 insertions(+), 24 deletions(-)
diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
index 1bede2e..d2b0964 100644
--- a/libglusterfs/src/globals.h
+++ b/libglusterfs/src/globals.h
@@ -111,6 +111,8 @@
#define GD_OP_VERSION_3_13_3 31303 /* Op-version for GlusterFS 3.13.3 */
+#define GD_OP_VERSION_6_0 60000 /* Op-version for GlusterFS 6.0 */
+
/* Downstream only change */
#define GD_OP_VERSION_3_11_2 31102 /* Op-version for RHGS 3.3.1-async */
#define GD_OP_VERSION_3_13_3 31303 /* Op-version for RHGS-3.4-Batch Update-1*/
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 7486f51..90eaa95 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -28,6 +28,7 @@
#include "glusterd-sm.h"
#include "glusterd-op-sm.h"
#include "glusterd-utils.h"
+#include "glusterd-mgmt.h"
#include "glusterd-server-quorum.h"
#include "glusterd-store.h"
#include "glusterd-locks.h"
@@ -3065,10 +3066,13 @@ __glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
int32_t op = 0;
char err_str[2048] = {0,};
xlator_t *this = NULL;
+ glusterd_conf_t *conf = NULL;
GF_ASSERT (req);
this = THIS;
GF_ASSERT (this);
+ conf = this->private;
+ GF_VALIDATE_OR_GOTO(this->name, conf, out);
ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
if (ret < 0) {
@@ -3109,12 +3113,18 @@ __glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
goto out;
}
- ret = glusterd_op_begin (req, cli_op, dict, err_str, sizeof (err_str));
+ if (conf->op_version < GD_OP_VERSION_6_0) {
+ gf_msg_debug(this->name, 0, "The cluster is operating at "
+ "version less than %d. Falling back to op-sm "
+ "framework.", GD_OP_VERSION_6_0);
+ ret = glusterd_op_begin(req, cli_op, dict, err_str, sizeof(err_str));
+ glusterd_friend_sm();
+ glusterd_op_sm();
+ } else {
+ ret = glusterd_mgmt_v3_initiate_profile_phases(req, cli_op, dict);
+ }
out:
- glusterd_friend_sm ();
- glusterd_op_sm ();
-
free (cli_req.dict.dict_val);
if (ret) {
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
index d7da3c1..751d6e4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
@@ -19,6 +19,7 @@
#include "glusterd-locks.h"
#include "glusterd-mgmt.h"
#include "glusterd-op-sm.h"
+#include "glusterd-server-quorum.h"
#include "glusterd-volgen.h"
#include "glusterd-store.h"
#include "glusterd-snapshot-utils.h"
@@ -213,6 +214,16 @@ gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
}
break;
+ case GD_OP_PROFILE_VOLUME:
+ ret = glusterd_op_stage_stats_volume(dict, op_errstr);
+ if (ret) {
+ gf_msg(this->name, GF_LOG_WARNING, 0,
+ GD_MSG_PRE_VALIDATION_FAIL,
+ "prevalidation failed for profile operation.");
+ goto out;
+ }
+ break;
+
case GD_OP_MAX_OPVERSION:
ret = 0;
break;
@@ -252,6 +263,16 @@ gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict,
}
break;
}
+ case GD_OP_PROFILE_VOLUME:
+ {
+ ret = gd_brick_op_phase(op, rsp_dict, dict, op_errstr);
+ if (ret) {
+ gf_log(this->name, GF_LOG_WARNING, "%s brickop failed",
+ gd_op_list[op]);
+ goto out;
+ }
+ break;
+ }
default:
break;
}
@@ -406,6 +427,17 @@ gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
break;
}
+ case GD_OP_PROFILE_VOLUME:
+ {
+ ret = glusterd_op_stats_volume(dict, op_errstr, rsp_dict);
+ if (ret) {
+ gf_msg(this->name, GF_LOG_ERROR, 0,
+ GD_MSG_COMMIT_OP_FAIL, "commit failed "
+ "volume profile operation.");
+ goto out;
+ }
+ break;
+ }
default:
break;
@@ -847,6 +879,7 @@ glusterd_pre_validate_aggr_rsp_dict (glusterd_op_t op,
case GD_OP_DETACH_TIER_STATUS:
case GD_OP_TIER_START_STOP:
case GD_OP_REMOVE_TIER_BRICK:
+ case GD_OP_PROFILE_VOLUME:
break;
case GD_OP_MAX_OPVERSION:
break;
@@ -1039,6 +1072,16 @@ glusterd_mgmt_v3_pre_validate (glusterd_op_t op, dict_t *req_dict,
goto out;
}
+ if (op == GD_OP_PROFILE_VOLUME) {
+ ret = glusterd_validate_quorum(this, op, req_dict, op_errstr);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_SERVER_QUORUM_NOT_MET, "Server quorum "
+ "not met. Rejecting operation.");
+ goto out;
+ }
+ }
+
/* Pre Validation on local node */
ret = gd_mgmt_v3_pre_validate_fn (op, req_dict, op_errstr,
rsp_dict, op_errno);
@@ -1157,6 +1200,7 @@ glusterd_mgmt_v3_build_payload (dict_t **req, char **op_errstr, dict_t *dict,
case GD_OP_REPLACE_BRICK:
case GD_OP_RESET_BRICK:
case GD_OP_ADD_TIER_BRICK:
+ case GD_OP_PROFILE_VOLUME:
{
ret = dict_get_str (dict, "volname", &volname);
if (ret) {
@@ -1309,12 +1353,11 @@ out:
}
int
-glusterd_mgmt_v3_brick_op (glusterd_op_t op, dict_t *req_dict, char **op_errstr,
- uint32_t txn_generation)
+glusterd_mgmt_v3_brick_op (glusterd_op_t op, dict_t *rsp_dict, dict_t *req_dict,
+ char **op_errstr, uint32_t txn_generation)
{
int32_t ret = -1;
int32_t peer_cnt = 0;
- dict_t *rsp_dict = NULL;
glusterd_peerinfo_t *peerinfo = NULL;
struct syncargs args = {0};
uuid_t peer_uuid = {0};
@@ -1329,14 +1372,6 @@ glusterd_mgmt_v3_brick_op (glusterd_op_t op, dict_t *req_dict, char **op_errstr,
GF_ASSERT (req_dict);
GF_ASSERT (op_errstr);
- rsp_dict = dict_new ();
- if (!rsp_dict) {
- gf_msg (this->name, GF_LOG_ERROR, 0,
- GD_MSG_DICT_CREATE_FAIL,
- "Failed to create response dictionary");
- goto out;
- }
-
/* Perform brick op on local node */
ret = gd_mgmt_v3_brick_op_fn (op, req_dict, op_errstr,
rsp_dict);
@@ -1361,11 +1396,8 @@ glusterd_mgmt_v3_brick_op (glusterd_op_t op, dict_t *req_dict, char **op_errstr,
goto out;
}
- dict_unref (rsp_dict);
- rsp_dict = NULL;
-
/* Sending brick op req to other nodes in the cluster */
- gd_syncargs_init (&args, NULL);
+ gd_syncargs_init (&args, rsp_dict);
synctask_barrier_init((&args));
peer_cnt = 0;
@@ -2108,6 +2140,180 @@ out:
}
int32_t
+glusterd_mgmt_v3_initiate_profile_phases (rpcsvc_request_t *req,
+ glusterd_op_t op, dict_t *dict)
+{
+ int32_t ret = -1;
+ int32_t op_ret = -1;
+ dict_t *req_dict = NULL;
+ dict_t *tmp_dict = NULL;
+ glusterd_conf_t *conf = NULL;
+ char *op_errstr = NULL;
+ xlator_t *this = NULL;
+ gf_boolean_t is_acquired = _gf_false;
+ uuid_t *originator_uuid = NULL;
+ uint32_t txn_generation = 0;
+ uint32_t op_errno = 0;
+
+ this = THIS;
+ GF_ASSERT (this);
+ GF_ASSERT (req);
+ GF_ASSERT (dict);
+ conf = this->private;
+ GF_ASSERT (conf);
+
+ /* Save the peer list generation */
+ txn_generation = conf->generation;
+ cmm_smp_rmb ();
+ /* This read memory barrier makes sure that this assignment happens here
+ * only and is not reordered and optimized by either the compiler or the
+ * processor.
+ */
+
+ /* Save the MY_UUID as the originator_uuid. This originator_uuid
+ * will be used by is_origin_glusterd() to determine if a node
+ * is the originator node for a command. */
+ originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+ gf_common_mt_uuid_t);
+ if (!originator_uuid) {
+ ret = -1;
+ goto out;
+ }
+
+ gf_uuid_copy (*originator_uuid, MY_UUID);
+ ret = dict_set_bin (dict, "originator_uuid",
+ originator_uuid, sizeof (uuid_t));
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_DICT_SET_FAILED,
+ "Failed to set originator_uuid.");
+ GF_FREE (originator_uuid);
+ goto out;
+ }
+
+ /* Marking the operation as complete synctasked */
+ ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_DICT_SET_FAILED,
+ "Failed to set synctasked flag.");
+ goto out;
+ }
+
+ /* Use a copy at local unlock as cli response will be sent before
+ * the unlock and the volname in the dict might be removed */
+ tmp_dict = dict_new();
+ if (!tmp_dict) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_DICT_CREATE_FAIL, "Unable to create dict");
+ goto out;
+ }
+ dict_copy (dict, tmp_dict);
+
+ /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+ ret = glusterd_mgmt_v3_initiate_lockdown (op, dict, &op_errstr,
+ &op_errno, &is_acquired,
+ txn_generation);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_MGMTV3_LOCKDOWN_FAIL,
+ "mgmt_v3 lockdown failed.");
+ goto out;
+ }
+
+ /* BUILD PAYLOAD */
+ ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL, LOGSTR_BUILD_PAYLOAD,
+ gd_op_list[op]);
+ if (op_errstr == NULL)
+ gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+ goto out;
+ }
+
+ /* PRE-COMMIT VALIDATE PHASE */
+ ret = glusterd_mgmt_v3_pre_validate (op, req_dict, &op_errstr,
+ &op_errno, txn_generation);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_PRE_VALIDATION_FAIL, "Pre Validation Failed");
+ goto out;
+ }
+
+ /* BRICK-OPS */
+ ret = glusterd_mgmt_v3_brick_op(op, dict, req_dict, &op_errstr,
+ txn_generation);
+ if (ret) {
+ gf_log(this->name, GF_LOG_ERROR, "Brick Op Failed");
+ goto out;
+ }
+
+ /* COMMIT OP PHASE */
+ ret = glusterd_mgmt_v3_commit (op, dict, req_dict, &op_errstr,
+ &op_errno, txn_generation);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_COMMIT_OP_FAIL, "Commit Op Failed");
+ goto out;
+ }
+
+ /* POST-COMMIT VALIDATE PHASE */
+ /* As of now, post_validate is not trying to cleanup any failed
+ commands. So as of now, I am sending 0 (op_ret as 0).
+ */
+ ret = glusterd_mgmt_v3_post_validate (op, 0, dict, req_dict, &op_errstr,
+ txn_generation);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_POST_VALIDATION_FAIL, "Post Validation Failed");
+ goto out;
+ }
+
+ ret = 0;
+out:
+ op_ret = ret;
+ /* UNLOCK PHASE FOR PEERS*/
+ (void) glusterd_mgmt_v3_release_peer_locks (op, dict, op_ret,
+ &op_errstr, is_acquired,
+ txn_generation);
+
+ /* LOCAL VOLUME(S) UNLOCK */
+ if (is_acquired) {
+ /* Trying to release multiple mgmt_v3 locks */
+ ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ GD_MSG_MGMTV3_UNLOCK_FAIL,
+ "Failed to release mgmt_v3 locks on localhost");
+ op_ret = ret;
+ }
+ }
+
+ if (op_ret && (op_errno == 0))
+ op_errno = EG_INTRNL;
+
+ if (op != GD_OP_MAX_OPVERSION) {
+ /* SEND CLI RESPONSE */
+ glusterd_op_send_cli_response (op, op_ret, op_errno, req,
+ dict, op_errstr);
+ }
+
+ if (req_dict)
+ dict_unref (req_dict);
+
+ if (tmp_dict)
+ dict_unref (tmp_dict);
+
+ if (op_errstr) {
+ GF_FREE (op_errstr);
+ op_errstr = NULL;
+ }
+
+ return 0;
+}
+
+int32_t
glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
dict_t *dict)
{
@@ -2465,7 +2671,7 @@ glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
goto out;
}
- ret = glusterd_mgmt_v3_brick_op (op, req_dict, &op_errstr,
+ ret = glusterd_mgmt_v3_brick_op (op, dict, req_dict, &op_errstr,
txn_generation);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
@@ -2526,7 +2732,7 @@ unbarrier:
goto out;
}
- ret = glusterd_mgmt_v3_brick_op (op, req_dict, &op_errstr,
+ ret = glusterd_mgmt_v3_brick_op (op, dict, req_dict, &op_errstr,
txn_generation);
if (ret) {
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
index 2215f17..eff070d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mgmt.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
@@ -37,7 +37,11 @@ glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
dict_t *dict);
int32_t
-glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
+glusterd_mgmt_v3_initiate_profile_phases(rpcsvc_request_t *req,
+ glusterd_op_t op, dict_t *dict);
+
+int32_t
+glusterd_mgmt_v3_initiate_snap_phases(rpcsvc_request_t *req, glusterd_op_t op,
dict_t *dict);
int
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 52a3db0..9f76ab3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -2032,7 +2032,7 @@ glusterd_op_stage_status_volume (dict_t *dict, char **op_errstr)
return ret;
}
-static int
+int
glusterd_op_stage_stats_volume (dict_t *dict, char **op_errstr)
{
int ret = -1;
@@ -3322,7 +3322,7 @@ glusterd_remove_profile_volume_options (glusterd_volinfo_t *volinfo)
dict_del (volinfo->dict, fd_stats_key);
}
-static int
+int
glusterd_op_stats_volume (dict_t *dict, char **op_errstr,
dict_t *rsp_dict)
{
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
index f2aee9c..e64d368 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
@@ -312,4 +312,10 @@ glusterd_dict_set_volid (dict_t *dict, char *volname, char **op_errstr);
int32_t
glusterd_tier_op (xlator_t *this, void *data);
+
+int
+glusterd_op_stats_volume (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int
+glusterd_op_stage_stats_volume (dict_t *dict, char **op_errstr);
#endif
--
1.8.3.1