From 23ab7175e64ab4d75fbcb6874008843cc78b65b8 Mon Sep 17 00:00:00 2001

From: Ashish Pandey <aspandey@redhat.com>

Date: Fri, 16 Apr 2021 18:48:56 +0530

Subject: [PATCH 541/542] glusterd-volgen: Add functionality to accept any

 custom xlator

Add new function which allow users to insert any custom xlators.

It makes to provide a way to add any processing into file operations.

Users can deploy the plugin(xlator shared object) and integrate it to glusterfsd.

If users want to enable a custom xlator, do the follows:

1. put xlator object(.so file) into "XLATOR_DIR/user/"

2. set the option user.xlator.<xlator> to the existing xlator-name to specify of the position in graph

3. restart gluster volume

Options for custom xlator are able to set in "user.xlator.<xlator>.<optkey>".

Backport of :

>https://github.com/gluster/glusterfs/commit/ea86b664f3b1f54901ce1b7d7fba7d80456f2089

>Fixes: https://github.com/gluster/glusterfs/issues/1943

>Change-Id: Ife3ae1514ea474f5dae2897223012f9d04b64674

>Signed-off-by:Ryo Furuhashi <ryo.furuhashi.nh@hitachi.com>

>Co-authored-by: Yaniv Kaul <ykaul@redhat.com>

>Co-authored-by: Xavi Hernandez <xhernandez@users.noreply.github.com>

Change-Id: Ic8f28bfcfde67213eb1092b0ebf4822c874d37bb

BUG: 1927235

Signed-off-by: Ashish Pandey <aspandey@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/236830

Tested-by: RHGS Build Bot <nigelb@redhat.com>

Reviewed-by: Ravishankar Narayanankutty <ravishankar@redhat.com>

Reviewed-by: Xavi Hernandez Juan <xhernandez@redhat.com>

---

 cli/src/cli-rpc-ops.c                       | 148 ++++++++++++++++++++------

 cli/src/cli.h                               |   2 -

 tests/basic/user-xlator.t                   |  65 ++++++++++++

 tests/env.rc.in                             |   3 +

 xlators/mgmt/glusterd/src/glusterd-volgen.c | 155 ++++++++++++++++++++++++++++

 5 files changed, 342 insertions(+), 31 deletions(-)

 create mode 100755 tests/basic/user-xlator.t

diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c

index 4e91265..51b5447 100644

--- a/cli/src/cli-rpc-ops.c

+++ b/cli/src/cli-rpc-ops.c

@@ -2269,49 +2269,131 @@ out:

     return ret;

 }

-char *

-is_server_debug_xlator(void *myframe)

+/*

+ * returns

+ *   1 : is server debug xlator

+ *   0 : is not server debug xlator

+ *  <0 : error

+ */

+static int

+is_server_debug_xlator(char *key, char *value)

+{

+    if (!key || !value)

+        return -1;

+

+    if (strcmp("debug.trace", key) == 0 ||

+        strcmp("debug.error-gen", key) == 0) {

+        if (strcmp("client", value) == 0)

+            return 0;

+        else

+            return 1;

+    }

+

+    return 0;

+}

+

+/*

+ * returns

+ *   1 : is user xlator

+ *   0 : is not user xlator

+ *  <0 : error

+ */

+static int

+is_server_user_xlator(char *key, char *value)

+{

+    int ret = 0;

+

+    if (!key || !value)

+        return -1;

+

+    ret = fnmatch("user.xlator.*", key, 0);

+    if (ret < 0) {

+        ret = -1;

+        goto out;

+    } else if (ret == FNM_NOMATCH) {

+        ret = 0;

+        goto out;

+    }

+

+    ret = fnmatch("user.xlator.*.*", key, 0);

+    if (ret < 0) {

+        ret = -1;

+        goto out;

+    } else if (ret != FNM_NOMATCH) {  // this is user xlator's option key

+        ret = 0;

+        goto out;

+    }

+

+    ret = 1;

+

+out:

+    return ret;

+}

+

+static int

+added_server_xlator(void *myframe, char **added_xlator)

 {

     call_frame_t *frame = NULL;

     cli_local_t *local = NULL;

     char **words = NULL;

     char *key = NULL;

     char *value = NULL;

-    char *debug_xlator = NULL;

+    int ret = 0;

     frame = myframe;

     local = frame->local;

     words = (char **)local->words;

     while (*words != NULL) {

-        if (strstr(*words, "trace") == NULL &&

-            strstr(*words, "error-gen") == NULL) {

-            words++;

-            continue;

-        }

-

         key = *words;

         words++;

         value = *words;

-        if (value == NULL)

+

+        if (!value) {

             break;

-        if (strstr(value, "client")) {

-            words++;

-            continue;

-        } else {

-            if (!(strstr(value, "posix") || strstr(value, "acl") ||

-                  strstr(value, "locks") || strstr(value, "io-threads") ||

-                  strstr(value, "marker") || strstr(value, "index"))) {

-                words++;

-                continue;

-            } else {

-                debug_xlator = gf_strdup(key);

-                break;

+        }

+

+        ret = is_server_debug_xlator(key, value);

+        if (ret < 0) {

+            gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,

+                   "failed to check that debug xlator was added");

+            ret = -1;

+            goto out;

+        }

+

+        if (ret) {

+            *added_xlator = gf_strdup(key);

+            if (!*added_xlator) {

+                gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,

+                       "Out of memory");

+                ret = -1;

+                goto out;

+            }

+            break;

+        }

+

+        ret = is_server_user_xlator(key, value);

+        if (ret < 0) {

+            gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,

+                   "failed to check that user xlator was added");

+            ret = -1;

+            goto out;

+        }

+

+        if (ret) {

+            *added_xlator = gf_strdup(key);

+            if (!*added_xlator) {

+                gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,

+                       "Out of memory");

+                ret = -1;

+                goto out;

             }

+            break;

         }

     }

-    return debug_xlator;

+out:

+    return ret;

 }

 int

@@ -2327,7 +2409,7 @@ gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,

     char msg[1024] = {

         0,

     };

-    char *debug_xlator = NULL;

+    char *added_xlator = NULL;

     char tmp_str[512] = {

         0,

     };

@@ -2365,18 +2447,26 @@ gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,

      * The process has to be restarted. So this is a check from the

      * volume set option such that if debug xlators such as trace/errorgen

      * are provided in the set command, warn the user.

+     * volume set option such that if user custom xlators or debug

+     * xlators such as trace/errorgen are provided in the set command,

+     * warn the user.

      */

-    debug_xlator = is_server_debug_xlator(myframe);

+    ret = added_server_xlator(myframe, &added_xlator);

+    if (ret < 0) {

+        gf_log("cli", GF_LOG_ERROR,

+               "failed to check that server graph has been changed");

+        goto out;

+    }

     if (dict_get_str(dict, "help-str", &help_str) && !msg[0])

         snprintf(msg, sizeof(msg), "Set volume %s",

                  (rsp.op_ret) ? "unsuccessful" : "successful");

-    if (rsp.op_ret == 0 && debug_xlator) {

+    if (rsp.op_ret == 0 && added_xlator) {

         snprintf(tmp_str, sizeof(tmp_str),

                  "\n%s translator has been "

                  "added to the server volume file. Please restart the"

                  " volume for enabling the translator",

-                 debug_xlator);

+                 added_xlator);

     }

     if ((global_state->mode & GLUSTER_MODE_XML) && (help_str == NULL)) {

@@ -2394,7 +2484,7 @@ gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,

             cli_err("volume set: failed");

     } else {

         if (help_str == NULL) {

-            if (debug_xlator == NULL)

+            if (added_xlator == NULL)

                 cli_out("volume set: success");

             else

                 cli_out("volume set: success%s", tmp_str);

@@ -2408,7 +2498,7 @@ gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,

 out:

     if (dict)

         dict_unref(dict);

-    GF_FREE(debug_xlator);

+    GF_FREE(added_xlator);

     cli_cmd_broadcast_response(ret);

     gf_free_xdr_cli_rsp(rsp);

     return ret;

diff --git a/cli/src/cli.h b/cli/src/cli.h

index 7b4f446..b5b69ea 100644

--- a/cli/src/cli.h

+++ b/cli/src/cli.h

@@ -502,8 +502,6 @@ cli_xml_output_snapshot(int cmd_type, dict_t *dict, int op_ret, int op_errno,

 int

 cli_xml_snapshot_status_single_snap(cli_local_t *local, dict_t *dict,

                                     char *key);

-char *

-is_server_debug_xlator(void *myframe);

 int32_t

 cli_cmd_snapshot_parse(const char **words, int wordcount, dict_t **options,

diff --git a/tests/basic/user-xlator.t b/tests/basic/user-xlator.t

new file mode 100755

index 0000000..a711f9f

--- /dev/null

+++ b/tests/basic/user-xlator.t

@@ -0,0 +1,65 @@

+#!/bin/bash

+

+. $(dirname $0)/../include.rc

+. $(dirname $0)/../volume.rc

+

+#### patchy.dev.d-backends-patchy1.vol

+brick=${B0//\//-}

+SERVER_VOLFILE="/var/lib/glusterd/vols/${V0}/${V0}.${H0}.${brick:1}-${V0}1.vol"

+

+cleanup;

+

+TEST mkdir -p $B0/single-brick

+TEST mkdir -p ${GLUSTER_XLATOR_DIR}/user

+

+## deploy dummy user xlator

+TEST cp ${GLUSTER_XLATOR_DIR}/playground/template.so ${GLUSTER_XLATOR_DIR}/user/hoge.so

+

+TEST glusterd

+TEST $CLI volume create $V0 replica 3  $H0:$B0/${V0}{1,2,3,4,5,6};

+TEST $CLI volume set $V0 user.xlator.hoge posix

+TEST grep -q 'user/hoge' ${SERVER_VOLFILE}

+

+TEST $CLI volume set $V0 user.xlator.hoge.opt1 10

+TEST grep -q '"option opt1 10"' ${SERVER_VOLFILE}

+TEST $CLI volume set $V0 user.xlator.hoge.opt2 hogehoge

+TEST grep -q '"option opt2 hogehoge"' ${SERVER_VOLFILE}

+TEST $CLI volume set $V0 user.xlator.hoge.opt3 true

+TEST grep -q '"option opt3 true"' ${SERVER_VOLFILE}

+

+TEST $CLI volume start $V0

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}3

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}4

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}5

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}6

+

+TEST $CLI volume set $V0 user.xlator.hoge trash

+TEST grep -q 'user/hoge' ${SERVER_VOLFILE}

+

+TEST $CLI volume stop $V0

+TEST $CLI volume start $V0

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}3

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}4

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}5

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}6

+

+TEST ! $CLI volume set $V0 user.xlator.hoge unknown

+TEST grep -q 'user/hoge' ${SERVER_VOLFILE} # When the CLI fails, the volfile is not modified.

+

+TEST $CLI volume stop $V0

+TEST $CLI volume start $V0

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}3

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}4

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}5

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}6

+

+#### teardown

+

+TEST rm -f ${GLUSTER_XLATOR_DIR}/user/hoge.so

+cleanup;

diff --git a/tests/env.rc.in b/tests/env.rc.in

index c7472a7..1f0ca88 100644

--- a/tests/env.rc.in

+++ b/tests/env.rc.in

@@ -40,3 +40,6 @@ export GLUSTER_LIBEXECDIR

 RUN_NFS_TESTS=@BUILD_GNFS@

 export RUN_NFS_TESTS

+

+GLUSTER_XLATOR_DIR=@libdir@/glusterfs/@PACKAGE_VERSION@/xlator

+export GLUSTER_XLATOR_DIR

\ No newline at end of file

diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c

index 1920284..a242b5c 100644

--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c

+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c

@@ -45,6 +45,11 @@ struct gd_validate_reconf_opts {

 extern struct volopt_map_entry glusterd_volopt_map[];

+struct check_and_add_user_xlator_t {

+    volgen_graph_t *graph;

+    char *volname;

+};

+

 #define RPC_SET_OPT(XL, CLI_OPT, XLATOR_OPT, ERROR_CMD)                        \

     do {                                                                       \

         char *_value = NULL;                                                   \

@@ -2822,6 +2827,145 @@ out:

     return ret;

 }

+static gf_boolean_t

+check_user_xlator_position(dict_t *dict, char *key, data_t *value,

+                           void *prev_xlname)

+{

+    if (strncmp(key, "user.xlator.", SLEN("user.xlator.")) != 0) {

+        return false;

+    }

+

+    if (fnmatch("user.xlator.*.*", key, 0) == 0) {

+        return false;

+    }

+

+    char *value_str = data_to_str(value);

+    if (!value_str) {

+        return false;

+    }

+

+    if (strcmp(value_str, prev_xlname) == 0) {

+        gf_log("glusterd", GF_LOG_INFO,

+               "found insert position of user-xlator(%s)", key);

+        return true;

+    }

+

+    return false;

+}

+

+static int

+set_user_xlator_option(dict_t *set_dict, char *key, data_t *value, void *data)

+{

+    xlator_t *xl = data;

+    char *optname = strrchr(key, '.') + 1;

+

+    gf_log("glusterd", GF_LOG_DEBUG, "set user xlator option %s = %s", key,

+           value->data);

+

+    return xlator_set_option(xl, optname, strlen(optname), data_to_str(value));

+}

+

+static int

+insert_user_xlator_to_graph(dict_t *set_dict, char *key, data_t *value,

+                            void *action_data)

+{

+    int ret = -1;

+

+    struct check_and_add_user_xlator_t *data = action_data;

+

+    char *xlator_name = strrchr(key, '.') + 1;  // user.xlator.<xlator_name>

+    char *xlator_option_matcher = NULL;

+    char *type = NULL;

+    xlator_t *xl = NULL;

+

+    // convert optkey to xlator type

+    if (gf_asprintf(&type, "user/%s", xlator_name) < 0) {

+        gf_log("glusterd", GF_LOG_ERROR, "failed to generate user-xlator type");

+        goto out;

+    }

+

+    gf_log("glusterd", GF_LOG_INFO, "add user xlator=%s to graph", type);

+

+    xl = volgen_graph_add(data->graph, type, data->volname);

+    if (!xl) {

+        goto out;

+    }

+

+    ret = gf_asprintf(&xlator_option_matcher, "user.xlator.%s.*", xlator_name);

+    if (ret < 0) {

+        gf_log("glusterd", GF_LOG_ERROR,

+               "failed to generate user-xlator option matcher");

+        goto out;

+    }

+

+    dict_foreach_fnmatch(set_dict, xlator_option_matcher,

+                         set_user_xlator_option, xl);

+

+out:

+    if (type)

+        GF_FREE(type);

+    if (xlator_option_matcher)

+        GF_FREE(xlator_option_matcher);

+

+    return ret;

+}

+

+static int

+validate_user_xlator_position(dict_t *this, char *key, data_t *value,

+                              void *unused)

+{

+    int ret = -1;

+    int i = 0;

+

+    if (!value)

+        goto out;

+

+    if (fnmatch("user.xlator.*.*", key, 0) == 0) {

+        ret = 0;

+        goto out;

+    }

+

+    char *value_str = data_to_str(value);

+    if (!value_str)

+        goto out;

+

+    int num_xlators = sizeof(server_graph_table) /

+                      sizeof(server_graph_table[0]);

+    for (i = 0; i < num_xlators; i++) {

+        if (server_graph_table[i].dbg_key &&

+            strcmp(value_str, server_graph_table[i].dbg_key) == 0) {

+            ret = 0;

+            goto out;

+        }

+    }

+

+out:

+    if (ret == -1)

+        gf_log("glusterd", GF_LOG_ERROR, "invalid user xlator position %s = %s",

+               key, value->data);

+

+    return ret;

+}

+

+static int

+check_and_add_user_xl(volgen_graph_t *graph, dict_t *set_dict, char *volname,

+                      char *prev_xlname)

+{

+    if (!prev_xlname)

+        goto out;

+

+    struct check_and_add_user_xlator_t data = {.graph = graph,

+                                               .volname = volname};

+

+    if (dict_foreach_match(set_dict, check_user_xlator_position, prev_xlname,

+                           insert_user_xlator_to_graph, &data) < 0) {

+        return -1;

+    }

+

+out:

+    return 0;

+}

+

 static int

 server_graph_builder(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,

                      dict_t *set_dict, void *param)

@@ -2831,6 +2975,12 @@ server_graph_builder(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,

     char *loglevel = NULL;

     int i = 0;

+    if (dict_foreach_fnmatch(set_dict, "user.xlator.*",

+                             validate_user_xlator_position, NULL) < 0) {

+        ret = -EINVAL;

+        goto out;

+    }

+

     i = sizeof(server_graph_table) / sizeof(server_graph_table[0]) - 1;

     while (i >= 0) {

@@ -2848,6 +2998,11 @@ server_graph_builder(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,

         if (ret)

             goto out;

+        ret = check_and_add_user_xl(graph, set_dict, volinfo->volname,

+                                    server_graph_table[i].dbg_key);

+        if (ret)

+            goto out;

+

         i--;

     }

-- 

1.8.3.1