From 66b1bb65618f8205e8aa7a0229b0f003f82c5c81 Mon Sep 17 00:00:00 2001
From: Poornima G <pgurusid@redhat.com>
Date: Fri, 22 Jan 2016 11:44:21 -0500
Subject: [PATCH 315/361] glusterd: add a cli command to trigger a statedump on
a client
With this, we will be able to trigger statedumps on remote Gluster
clients, mainly targetted for applications using libgfapi.
Design:
SIGUSR signal is the most comman way of taking a statedump in Gluster.
But it cannot be used for libgfapi based processes, as the process
loading the library might have already consumed SIGUSR signal. Hence
going by the command way.
One has to issue a Gluster command to initiate a statedump on the
libgfapi based client. The command takes hostname and PID as an
argument. All the glusterds in the cluster, check if they are connected
to the specified hostname, and send an RPC request to all the connected
clients from that hostname (via the mgmt connection).
[ndevos: minor fixes and split patch in smaller pieces]
mainline:
> BUG: 1169302
> Reviewed-on: http://review.gluster.org/16357
> Reviewed-on: https://review.gluster.org/9228
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Niels de Vos <ndevos@redhat.com>
> Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
> Reviewed-by: Samikshan Bairagya <samikshan@gmail.com>
> Tested-by: Niels de Vos <ndevos@redhat.com>
(cherry picked from commit 1f1f1eee47c4aad678bd52971b96c1debfce4805)
BUG: 1378085
Change-Id: Icbe4d2f026b32a2c7d5535e1bfb2cdaaff042e91
Signed-off-by: Poornima G <pgurusid@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/101296
Tested-by: Milind Changire <mchangir@redhat.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
---
cli/src/cli-cmd-parser.c | 46 +++++++++++++----
cli/src/cli-cmd-volume.c | 4 +-
doc/debugging/statedump.md | 13 +++++
glusterfsd/src/glusterfsd-mgmt.c | 1 +
libglusterfs/src/common-utils.c | 21 ++++++++
libglusterfs/src/common-utils.h | 2 +
rpc/rpc-lib/src/protocol-common.h | 1 +
rpc/rpc-lib/src/rpc-transport.h | 2 +-
rpc/xdr/src/rpc-common-xdr.x | 3 ++
tests/bugs/cli/bug-1169302.t | 34 +++++++++++++
xlators/mgmt/glusterd/src/glusterd-utils.c | 43 ++++++++++++++++
xlators/mgmt/glusterd/src/glusterd-utils.h | 4 ++
xlators/mgmt/glusterd/src/glusterd-volume-ops.c | 7 +++
xlators/mgmt/glusterd/src/glusterd.c | 67 +++++++++++++++++++++++++
xlators/mgmt/glusterd/src/glusterd.h | 4 ++
15 files changed, 239 insertions(+), 13 deletions(-)
create mode 100755 tests/bugs/cli/bug-1169302.t
diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 2e61c3a..7637944 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -3530,19 +3530,44 @@ cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
dict_t *dict = NULL;
int option_cnt = 0;
char *option = NULL;
- char option_str[100] = {0,};
-
- for (i = 3; i < wordcount; i++, option_cnt++) {
- if (!cli_cmd_validate_dumpoption (words[i], &option)) {
+ char option_str[_POSIX_HOST_NAME_MAX + 100] = {0,};
+ char *tmp = NULL;
+ char *ip_addr = NULL;
+ char *pid = NULL;
+
+ if ((wordcount >= 5) && ((strcmp (words[3], "client")) == 0)) {
+ tmp = gf_strdup(words[4]);
+ if (!tmp) {
+ ret = -1;
+ goto out;
+ }
+ ip_addr = strtok(tmp, ":");
+ pid = strtok(NULL, ":");
+ if (valid_internet_address (ip_addr, _gf_true)
+ && pid && gf_valid_pid (pid, strlen(pid))) {
+ strncat (option_str, words[3], strlen (words[3]));
+ strncat (option_str, " ", 1);
+ strncat (option_str, ip_addr, strlen (ip_addr));
+ strncat (option_str, " ", 1);
+ strncat (option_str, pid, strlen (pid));
+ option_cnt = 3;
+ } else {
+ ret = -1;
+ goto out;
+ }
+ } else {
+ for (i = 3; i < wordcount; i++, option_cnt++) {
+ if (!cli_cmd_validate_dumpoption (words[i], &option)) {
+ ret = -1;
+ goto out;
+ }
+ strncat (option_str, option, strlen (option));
+ strncat (option_str, " ", 1);
+ }
+ if ((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
ret = -1;
goto out;
}
- strncat (option_str, option, strlen (option));
- strncat (option_str, " ", 1);
- }
- if((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
- ret = -1;
- goto out;
}
dict = dict_new ();
@@ -3559,6 +3584,7 @@ cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
*options = dict;
out:
+ GF_FREE (tmp);
if (ret && dict)
dict_destroy (dict);
if (ret)
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 04fe432..6737ffd 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -3295,8 +3295,8 @@ struct cli_cmd volume_cmds[] = {
cli_cmd_volume_heal_cbk,
"self-heal commands on volume specified by <VOLNAME>"},
- {"volume statedump <VOLNAME> [nfs|quotad] [all|mem|iobuf|callpool|priv|fd|"
- "inode|history]...",
+ {"volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|"
+ "priv|fd|inode|history]... | [client <hostname:process-id>]]",
cli_cmd_volume_statedump_cbk,
"perform statedump on bricks"},
diff --git a/doc/debugging/statedump.md b/doc/debugging/statedump.md
index f34a5c3..e4e02a9 100644
--- a/doc/debugging/statedump.md
+++ b/doc/debugging/statedump.md
@@ -19,6 +19,19 @@ For quotad: `gluster volume statedump <volname> quotad`
For brick-processes files will be created in `statedump-directory` with name of the file as `hyphenated-brick-path.<pid>.dump.timestamp`. For all other processes it will be `glusterdump.<pid>.dump.timestamp`.
+For applications using libgfapi, `SIGUSR1` cannot be used, eg: smbd/libvirtd
+processes could have used the `SIGUSR1` signal already for other purposes.
+To generate statedump for the processes, using libgfapi, below command can be
+executed from one of the nodes in the gluster cluster to which the libgfapi
+application is connected to.
+
+ gluster volume statedump <volname> client <hostname>:<process id>
+
+The statedumps can be found in the `statedump-directory`, the name of the
+statedumps being `glusterdump.<pid>.dump.timestamp`. For a process there can be
+multiple such files created depending on the number of times the volume is
+accessed by the process (related to the number of `glfs_init()` calls).
+
##How to read statedump
We shall see snippets of each type of statedump.
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index a12496e..bf7b4fc 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1450,6 +1450,7 @@ rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
[GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
[GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
mgmt_cbk_event},
+ [GF_CBK_STATEDUMP] = {"STATEDUMP", GF_CBK_STATEDUMP, mgmt_cbk_event},
};
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index d36069f..9d9f1d5 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -3670,6 +3670,27 @@ out:
return running;
}
+/* Check if the pid is > 0 */
+gf_boolean_t
+gf_valid_pid (const char *pid, int length)
+{
+ gf_boolean_t ret = _gf_true;
+ pid_t value = 0;
+ char *end_ptr = NULL;
+
+ if (length <= 0) {
+ ret = _gf_false;
+ goto out;
+ }
+
+ value = strtol (pid, &end_ptr, 10);
+ if (value <= 0) {
+ ret = _gf_false;
+ }
+out:
+ return ret;
+}
+
static int
dht_is_linkfile_key (dict_t *this, char *key, data_t *value, void *data)
{
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index 2a6d02c..ccda0d9 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -786,6 +786,8 @@ int gf_thread_create_detached (pthread_t *thread,
gf_boolean_t
gf_is_service_running (char *pidfile, int *pid);
+gf_boolean_t
+gf_valid_pid (const char *pid, int length);
int
gf_skip_header_section (int fd, int header_len);
diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h
index 0fe7d1f..ea680f9 100644
--- a/rpc/rpc-lib/src/protocol-common.h
+++ b/rpc/rpc-lib/src/protocol-common.h
@@ -141,6 +141,7 @@ enum gf_cbk_procnum {
GF_CBK_CHILD_UP,
GF_CBK_CHILD_DOWN,
GF_CBK_RECALL_LEASE,
+ GF_CBK_STATEDUMP,
GF_CBK_MAXVALUE,
};
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index 630729d..dbfa32a 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -70,7 +70,7 @@ struct peer_info {
uint32_t max_op_version;
uint32_t min_op_version;
//Volume mounted by client
- char volname[1024];
+ char volname[NAME_MAX];
};
typedef struct peer_info peer_info_t;
diff --git a/rpc/xdr/src/rpc-common-xdr.x b/rpc/xdr/src/rpc-common-xdr.x
index c2b2570..169b16d 100644
--- a/rpc/xdr/src/rpc-common-xdr.x
+++ b/rpc/xdr/src/rpc-common-xdr.x
@@ -24,6 +24,9 @@ struct gf_dump_req {
u_quad_t gfs_id;
};
+struct gf_statedump {
+ unsigned int pid;
+};
struct gf_prog_detail {
string progname<>;
diff --git a/tests/bugs/cli/bug-1169302.t b/tests/bugs/cli/bug-1169302.t
new file mode 100755
index 0000000..92252aa
--- /dev/null
+++ b/tests/bugs/cli/bug-1169302.t
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../cluster.rc
+
+function check_peers {
+ $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+cleanup;
+
+#setup cluster and test volume
+TEST launch_cluster 3; # start 3-node virtual cluster
+TEST $CLI_1 peer probe $H2; # peer probe server 2 from server 1 cli
+TEST $CLI_1 peer probe $H3; # peer probe server 3 from server 1 cli
+
+EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers;
+
+TEST $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0 $H3:$B3/$V0
+TEST $CLI_1 volume start $V0
+
+# there is no gfapi application to take statedumps yet, it will get added in
+# the next patch, this only tests the CLI for correctness
+
+cleanup_statedump
+
+TEST ! $CLI_1 volume statedump $V0 client $H2:0
+TEST ! $CLI_2 volume statedump $V0 client $H2:-1
+TEST $CLI_3 volume statedump $V0 client $H2:765
+TEST ! $CLI_1 volume statedump $V0 client $H2:
+TEST ! $CLI_2 volume statedump $V0 client
+
+cleanup_statedump
+cleanup;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index e976ba2..9e9d609 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -7022,6 +7022,49 @@ out:
}
int
+glusterd_client_statedump (char *volname, char *options, int option_cnt,
+ char **op_errstr)
+{
+ int ret = 0;
+ char *dup_options = NULL;
+ char *option = NULL;
+ char *tmpptr = NULL;
+ char msg[256] = {0,};
+ char *target_ip = NULL;
+ char *pid = NULL;
+
+ dup_options = gf_strdup (options);
+ option = strtok_r (dup_options, " ", &tmpptr);
+ if (strcmp (option, "client")) {
+ snprintf (msg, sizeof (msg), "for gluster client statedump, options "
+ "should be after the key 'client'");
+ *op_errstr = gf_strdup (msg);
+ ret = -1;
+ goto out;
+ }
+ target_ip = strtok_r (NULL, " ", &tmpptr);
+ if (target_ip == NULL) {
+ snprintf (msg, sizeof (msg), "ip address not specified");
+ *op_errstr = gf_strdup (msg);
+ ret = -1;
+ goto out;
+ }
+
+ pid = strtok_r (NULL, " ", &tmpptr);
+ if (pid == NULL) {
+ snprintf (msg, sizeof (msg), "pid not specified");
+ *op_errstr = gf_strdup (msg);
+ ret = -1;
+ goto out;
+ }
+
+ ret = glusterd_client_statedump_submit_req (volname, target_ip, pid);
+out:
+ GF_FREE (dup_options);
+ return ret;
+}
+
+int
glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr)
{
int ret = -1;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 160b2ef..b1493df 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -348,6 +348,10 @@ int
glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr);
int
+glusterd_client_statedump (char *volname, char *options, int option_cnt,
+ char **op_errstr);
+
+int
glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr);
gf_boolean_t
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index d157734..2eb7548 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2860,6 +2860,13 @@ glusterd_op_statedump_volume (dict_t *dict, char **op_errstr)
op_errstr);
if (ret)
goto out;
+
+ } else if (strstr (options, "client")) {
+ ret = glusterd_client_statedump (volname, options, option_cnt,
+ op_errstr);
+ if (ret)
+ goto out;
+
} else {
cds_list_for_each_entry (brickinfo, &volinfo->bricks,
brick_list) {
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 6599c53..ae81de6 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -47,6 +47,7 @@
#include "glusterd-geo-rep.h"
#include "run.h"
#include "rpc-clnt-ping.h"
+#include "rpc-common-xdr.h"
#include "syncop.h"
@@ -232,6 +233,72 @@ out:
}
int
+glusterd_client_statedump_submit_req (char *volname, char *target_ip,
+ char *pid)
+{
+ gf_statedump statedump_req = {0, };
+ glusterd_conf_t *conf = NULL;
+ int ret = 0;
+ char *end_ptr = NULL;
+ rpc_transport_t *trans = NULL;
+ char *ip_addr = NULL;
+ xlator_t *this = NULL;
+ char tmp[UNIX_PATH_MAX] = {0, };
+
+ this = THIS;
+ GF_ASSERT (this);
+ conf = this->private;
+ GF_ASSERT (conf);
+
+ if (target_ip == NULL || pid == NULL) {
+ ret = -1;
+ goto out;
+ }
+
+ statedump_req.pid = strtol (pid, &end_ptr, 10);
+
+ gf_msg_debug (this->name, 0, "Performing statedump on volume %s "
+ "client with pid:%d host:%s", volname, statedump_req.pid,
+ target_ip);
+
+ pthread_mutex_lock (&conf->xprt_lock);
+ {
+ list_for_each_entry (trans, &conf->xprt_list, list) {
+ /* check if this connection matches "all" or the
+ * volname */
+ if (strncmp (volname, "all", NAME_MAX) &&
+ strncmp (trans->peerinfo.volname, volname,
+ NAME_MAX)) {
+ /* no match, try next trans */
+ continue;
+ }
+
+ strcpy (tmp, trans->peerinfo.identifier);
+ ip_addr = strtok (tmp, ":");
+ if (gf_is_same_address (ip_addr, target_ip)) {
+ /* Every gluster client would have
+ * connected to glusterd(volfile server). This
+ * connection is used to send the statedump
+ * request rpc to the application.
+ */
+ gf_msg_trace (this->name, 0, "Submitting "
+ "statedump rpc request for %s",
+ trans->peerinfo.identifier);
+ rpcsvc_request_submit (conf->rpc, trans,
+ &glusterd_cbk_prog,
+ GF_CBK_STATEDUMP,
+ &statedump_req, this->ctx,
+ (xdrproc_t)xdr_gf_statedump);
+ }
+ }
+ }
+ pthread_mutex_unlock (&conf->xprt_lock);
+out:
+ return ret;
+
+}
+
+int
glusterd_fetchspec_notify (xlator_t *this)
{
int ret = -1;
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index c84f019..ab7c03e 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -963,6 +963,10 @@ glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
char *hostname, dict_t *dict);
int
+glusterd_client_statedump_submit_req (char *volname, char *target_ip,
+ char *pid);
+
+int
glusterd_fetchspec_notify (xlator_t *this);
int
--
1.8.3.1