From 2b95b8fad1c35aaa995c664cf668a9be0d8c621b Mon Sep 17 00:00:00 2001

From: Anuradha Talur <atalur@redhat.com>

Date: Wed, 16 Mar 2016 10:55:09 +0530

Subject: [PATCH 44/80] glusterd / afr : Enable auto heal when replica count increases

        Backport of http://review.gluster.org/13806

In replicate volumes, when a brick is added to a replicate

group, heal to the new brick should be triggered.

Also, the new brick should not be considered as source for

healing till it is up to date.

Previously, extended attributes had to be set manually on

the bricks for this to happen. This patch is part 1 patch

to automate this process.

        >Change-Id: Ica83592aab8edbe49e2bb9d8d4824cf5c76324b7

        >BUG: 1320020

        >Reviewed-on: http://review.gluster.org/13806

        >Smoke: Gluster Build System <jenkins@build.gluster.com>

        >Tested-by: Anuradha Talur <atalur@redhat.com>

        >NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

        >CentOS-regression: Gluster Build System <jenkins@build.gluster.com>

        >Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

Change-Id: I183fd491d94534cf72e0bcc691911555b350fa17

BUG: 1248998

Signed-off-by: Anuradha Talur <atalur@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/71424

Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

---

 libglusterfs/src/globals.h                         |    2 +-

 libglusterfs/src/glusterfs.h                       |    1 +

 xlators/mgmt/glusterd/src/glusterd-brick-ops.c     |   86 ++++++++++---

 xlators/mgmt/glusterd/src/glusterd-replace-brick.c |   76 +----------

 xlators/mgmt/glusterd/src/glusterd-utils.c         |  141 ++++++++++++++++++++

 xlators/mgmt/glusterd/src/glusterd-utils.h         |   10 ++

 xlators/mgmt/glusterd/src/glusterd-volgen.c        |   52 +++++++

 xlators/mgmt/glusterd/src/glusterd-volgen.h        |    3 +

 8 files changed, 280 insertions(+), 91 deletions(-)

diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h

index ad0aef8..4fe4bcb 100644

--- a/libglusterfs/src/globals.h

+++ b/libglusterfs/src/globals.h

@@ -42,7 +42,7 @@

  */

 #define GD_OP_VERSION_MIN  1 /* MIN is the fresh start op-version, mostly

                                 should not change */

-#define GD_OP_VERSION_MAX  GD_OP_VERSION_3_7_7 /* MAX VERSION is the maximum

+#define GD_OP_VERSION_MAX  GD_OP_VERSION_3_7_10 /* MAX VERSION is the maximum

                                                   count in VME table, should

                                                   keep changing with

                                                   introduction of newer

diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h

index 81caf12..d1db087 100644

--- a/libglusterfs/src/glusterfs.h

+++ b/libglusterfs/src/glusterfs.h

@@ -176,6 +176,7 @@

 #define GF_AFR_SBRAIN_CHOICE "replica.split-brain-choice"

 #define GF_AFR_SPB_CHOICE_TIMEOUT "replica.split-brain-choice-timeout"

 #define GF_AFR_SBRAIN_RESOLVE "replica.split-brain-heal-finalize"

+#define GF_AFR_ADD_BRICK "trusted.add-brick"

 #define GF_AFR_REPLACE_BRICK "trusted.replace-brick"

 #define GF_AFR_DIRTY "trusted.afr.dirty"

diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c

index 859795b..540f8f7 100644

--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c

+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c

@@ -26,6 +26,7 @@

 #include "glusterd-messages.h"

 #include "glusterd-server-quorum.h"

 #include "run.h"

+#include "glusterd-volgen.h"

 #include <sys/signal.h>

 /* misc */

@@ -1238,6 +1239,7 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,

         char                         *brick_mount_dir  = NULL;

         xlator_t                     *this           = NULL;

         glusterd_conf_t              *conf           = NULL;

+        gf_boolean_t                  is_valid_add_brick = _gf_false;

         this = THIS;

         GF_ASSERT (this);

@@ -1325,6 +1327,7 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,

         /* Gets changed only if the options are given in add-brick cli */

         if (type)

                 volinfo->type = type;

+

         if (replica_count) {

                 volinfo->replica_count = replica_count;

         }

@@ -1360,6 +1363,27 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,

                         CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;

 #endif

+        /* This check needs to be added to distinguish between

+         * attach-tier commands and add-brick commands.

+         * When a tier is attached, adding is done via add-brick

+         * and setting of pending xattrs shouldn't be done for

+         * attach-tiers as they are virtually new volumes.

+         */

+        if (glusterd_is_volume_replicate (volinfo)) {

+                if (replica_count &&

+                    !dict_get (dict, "attach-tier") &&

+                    conf->op_version >= GD_OP_VERSION_3_7_10) {

+                        is_valid_add_brick = _gf_true;

+                        ret = generate_dummy_client_volfiles (volinfo);

+                        if (ret) {

+                                gf_msg (THIS->name, GF_LOG_ERROR, 0,

+                                        GD_MSG_VOLFILE_CREATE_FAIL,

+                                        "Failed to create volfile.");

+                                goto out;

+                                }

+                        }

+        }

+

         while (i <= count) {

                 ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,

                                                               &brickinfo);

@@ -1391,6 +1415,16 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,

                         }

                 }

+                /* if the volume is a replicate volume, do: */

+                if (is_valid_add_brick) {

+                        if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {

+                                ret = glusterd_handle_replicate_brick_ops (

+                                                           volinfo, brickinfo,

+                                                           GD_OP_ADD_BRICK);

+                                if (ret < 0)

+                                        goto out;

+                        }

+                }

                 ret = glusterd_brick_start (volinfo, brickinfo,

                                             _gf_true);

                 if (ret)

@@ -1519,22 +1553,6 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr, dict_t *rsp_dict)

         conf = this->private;

         GF_ASSERT (conf);

-        ret = dict_get_int32 (dict, "replica-count", &replica_count);

-        if (ret) {

-                gf_msg_debug (THIS->name, 0,

-                        "Unable to get replica count");

-        }

-

-        if (replica_count > 0) {

-                ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,

-                                        msg, sizeof(msg));

-                if (ret) {

-                        gf_msg (this->name, GF_LOG_ERROR, 0,

-                                GD_MSG_OP_VERSION_MISMATCH, "%s", msg);

-                        *op_errstr = gf_strdup (msg);

-                        goto out;

-                }

-        }

         ret = dict_get_str (dict, "volname", &volname);

         if (ret) {

                 gf_msg (THIS->name, GF_LOG_ERROR, errno,

@@ -1555,6 +1573,42 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr, dict_t *rsp_dict)

         if (ret)

                 goto out;

+        ret = dict_get_int32 (dict, "replica-count", &replica_count);

+        if (ret) {

+                gf_msg_debug (THIS->name, 0,

+                        "Unable to get replica count");

+        }

+

+        if (replica_count > 0) {

+                ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,

+                                        msg, sizeof(msg));

+                if (ret) {

+                        gf_msg (this->name, GF_LOG_ERROR, 0,

+                                GD_MSG_OP_VERSION_MISMATCH, "%s", msg);

+                        *op_errstr = gf_strdup (msg);

+                        goto out;

+                }

+        }

+

+        /* Do not allow add-brick for stopped volumes when replica-count

+         * is being increased.

+         */

+        if (glusterd_is_volume_replicate (volinfo)) {

+                if (conf->op_version >= GD_OP_VERSION_3_7_10 &&

+                    !dict_get (dict, "attach-tier") &&

+                    replica_count &&

+                    GLUSTERD_STATUS_STOPPED == volinfo->status) {

+                        ret = -1;

+                        snprintf (msg, sizeof (msg), " Volume must not be in"

+                                  " stopped state when replica-count needs to "

+                                  " be increased.");

+                        gf_msg (THIS->name, GF_LOG_ERROR, 0,

+                                GD_MSG_BRICK_ADD_FAIL, "%s", msg);

+                        *op_errstr = gf_strdup (msg);

+                        goto out;

+                }

+        }

+

         if (conf->op_version > GD_OP_VERSION_3_7_5 &&

             is_origin_glusterd (dict)) {

                 ret = glusterd_validate_quorum (this, GD_OP_ADD_BRICK, dict,

diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c

index d0a1126..ecfdc0a 100644

--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c

+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c

@@ -44,78 +44,6 @@ glusterd_mgmt_v3_initiate_replace_brick_cmd_phases (rpcsvc_request_t *req,

                                                     glusterd_op_t op,

                                                     dict_t *dict);

 int

-glusterd_handle_replicate_replace_brick (glusterd_volinfo_t *volinfo,

-                                         glusterd_brickinfo_t *brickinfo)

-{

-        int32_t                    ret               = -1;

-        char                       tmpmount[]        = "/tmp/mntXXXXXX";

-        char                       logfile[PATH_MAX] = {0,};

-        int                        dirty[3]          = {0,};

-        runner_t                   runner            = {0};

-        glusterd_conf_t           *priv              = NULL;

-        char                      *pid               = NULL;

-

-        priv = THIS->private;

-

-        dirty[2] = hton32(1);

-

-        ret = sys_lsetxattr (brickinfo->path, GF_AFR_DIRTY, dirty,

-                             sizeof (dirty), 0);

-        if (ret == -1) {

-                gf_msg (THIS->name, GF_LOG_ERROR, errno,

-                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"

-                        " attribute %s : %s.", GF_AFR_DIRTY, strerror (errno));

-                goto out;

-        }

-

-        if (mkdtemp (tmpmount) == NULL) {

-                gf_msg (THIS->name, GF_LOG_ERROR, errno,

-                        GD_MSG_DIR_OP_FAILED,

-                        "failed to create a temporary mount directory.");

-                ret = -1;

-                goto out;

-        }

-        snprintf (logfile, sizeof (logfile),

-                  DEFAULT_LOG_FILE_DIRECTORY"/%s-replace-brick-mount.log",

-                  volinfo->volname);

-

-        ret = gf_asprintf (&pid, "%d", GF_CLIENT_PID_SELF_HEALD);

-        if (ret < 0)

-                goto out;

-

-        runinit (&runner);

-        runner_add_args (&runner, SBIN_DIR"/glusterfs",

-                         "-s", "localhost",

-                         "--volfile-id", volinfo->volname,

-                         "--client-pid", pid,

-                         "-l", logfile, tmpmount, NULL);

-        synclock_unlock (&priv->big_lock);

-        ret = runner_run (&runner);

-

-        if (ret) {

-                runner_log (&runner, THIS->name, GF_LOG_ERROR, "mount command"

-                            "failed.");

-                goto lock;

-        }

-        ret = sys_lsetxattr (tmpmount, GF_AFR_REPLACE_BRICK,

-                             brickinfo->brick_id, sizeof (brickinfo->brick_id),

-                             0);

-        if (ret == -1)

-                gf_msg (THIS->name, GF_LOG_ERROR, errno,

-                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"

-                        " attribute %s : %s", GF_AFR_REPLACE_BRICK,

-                        strerror (errno));

-        gf_umount_lazy (THIS->name, tmpmount, 1);

-lock:

-        synclock_lock (&priv->big_lock);

-out:

-        if (pid)

-                GF_FREE (pid);

-        gf_msg_debug ("glusterd", 0, "Returning with ret");

-        return ret;

-}

-

-int

 __glusterd_handle_replace_brick (rpcsvc_request_t *req)

 {

         int32_t                         ret = -1;

@@ -659,8 +587,8 @@ glusterd_op_perform_replace_brick (glusterd_volinfo_t  *volinfo,

         /* if the volume is a replicate volume, do: */

         if (glusterd_is_volume_replicate (volinfo)) {

                 if (!gf_uuid_compare (new_brickinfo->uuid, MY_UUID)) {

-                        ret = glusterd_handle_replicate_replace_brick

-                                  (volinfo, new_brickinfo);

+                        ret = glusterd_handle_replicate_brick_ops (volinfo,

+                                        new_brickinfo, GD_OP_REPLACE_BRICK);

                         if (ret < 0)

                                 goto out;

                 }

diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c

index bc4f7ae..639404a 100644

--- a/xlators/mgmt/glusterd/src/glusterd-utils.c

+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c

@@ -7102,6 +7102,33 @@ void glusterd_update_tier_status (glusterd_volinfo_t *volinfo) {

 }

 int

+glusterd_get_dummy_client_filepath (char *filepath,

+                                    glusterd_volinfo_t *volinfo,

+                                    gf_transport_type type)

+{

+        int   ret             = 0;

+        char  path[PATH_MAX]  = {0,};

+

+        switch (type) {

+        case GF_TRANSPORT_TCP:

+        case GF_TRANSPORT_BOTH_TCP_RDMA:

+                snprintf (filepath, PATH_MAX,

+                          "/tmp/%s.tcp-fuse.vol", volinfo->volname);

+                break;

+

+        case GF_TRANSPORT_RDMA:

+                snprintf (filepath, PATH_MAX,

+                          "/tmp/%s.rdma-fuse.vol", volinfo->volname);

+                break;

+        default:

+                ret = -1;

+                break;

+        }

+

+        return ret;

+}

+

+int

 glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,

                               size_t len, int cmd, defrag_cbk_fn_t cbk)

 {

@@ -11057,3 +11084,117 @@ gd_get_shd_key (int type)

         }

         return key;

 }

+

+int

+glusterd_handle_replicate_brick_ops (glusterd_volinfo_t *volinfo,

+                                     glusterd_brickinfo_t *brickinfo,

+                                     glusterd_op_t op)

+{

+        int32_t                    ret               = -1;

+        char                       tmpmount[]        = "/tmp/mntXXXXXX";

+        char                       logfile[PATH_MAX] = {0,};

+        int                        dirty[3]          = {0,};

+        runner_t                   runner            = {0};

+        glusterd_conf_t           *priv              = NULL;

+        char                      *pid               = NULL;

+        char                       vpath[PATH_MAX]   = {0,};

+        char                      *volfileserver     = NULL;

+

+        priv = THIS->private;

+        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);

+

+        dirty[2] = hton32(1);

+

+        ret = sys_lsetxattr (brickinfo->path, GF_AFR_DIRTY, dirty,

+                             sizeof (dirty), 0);

+        if (ret == -1) {

+                gf_msg (THIS->name, GF_LOG_ERROR, errno,

+                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"

+                        " attribute %s : %s.", GF_AFR_DIRTY, strerror (errno));

+                goto out;

+        }

+

+        if (mkdtemp (tmpmount) == NULL) {

+                gf_msg (THIS->name, GF_LOG_ERROR, errno,

+                        GD_MSG_DIR_OP_FAILED,

+                        "failed to create a temporary mount directory.");

+                ret = -1;

+                goto out;

+        }

+

+        ret = gf_asprintf (&pid, "%d", GF_CLIENT_PID_SELF_HEALD);

+        if (ret < 0)

+                goto out;

+

+        switch (op) {

+        case GD_OP_REPLACE_BRICK:

+        if (dict_get_str (THIS->options, "transport.socket.bind-address",

+                          &volfileserver) != 0)

+                volfileserver = "localhost";

+

+                snprintf (logfile, sizeof (logfile),

+                          DEFAULT_LOG_FILE_DIRECTORY"/%s-replace-brick-mount.log",

+                          volinfo->volname);

+                if (!*logfile) {

+                        ret = -1;

+                        goto out;

+                }

+                runinit (&runner);

+                runner_add_args (&runner, SBIN_DIR"/glusterfs",

+                                 "-s", volfileserver,

+                                 "--volfile-id", volinfo->volname,

+                                 "--client-pid", pid,

+                                 "-l", logfile, tmpmount, NULL);

+                break;

+

+        case GD_OP_ADD_BRICK:

+                snprintf (logfile, sizeof (logfile),

+                          DEFAULT_LOG_FILE_DIRECTORY"/%s-add-brick-mount.log",

+                          volinfo->volname);

+                if (!*logfile) {

+                        ret = -1;

+                        goto out;

+                }

+                ret = glusterd_get_dummy_client_filepath (vpath, volinfo,

+                                                    volinfo->transport_type);

+                if (ret) {

+                        gf_log ("", GF_LOG_ERROR, "Failed to get "

+                                "volfile path");

+                        goto out;

+                }

+                runinit (&runner);

+                runner_add_args (&runner, SBIN_DIR"/glusterfs",

+                                 "--volfile", vpath,

+                                 "--client-pid", pid,

+                                 "-l", logfile, tmpmount, NULL);

+                break;

+        default:

+                break;

+        }

+        synclock_unlock (&priv->big_lock);

+        ret = runner_run (&runner);

+

+        if (ret) {

+                gf_log (THIS->name, GF_LOG_ERROR, "mount command"

+                        " failed.");

+                goto lock;

+        }

+        ret = sys_lsetxattr (tmpmount, (op == GD_OP_REPLACE_BRICK) ?

+                             GF_AFR_REPLACE_BRICK : GF_AFR_ADD_BRICK,

+                             brickinfo->brick_id, sizeof (brickinfo->brick_id),

+                             0);

+        if (ret == -1)

+                gf_msg (THIS->name, GF_LOG_ERROR, errno,

+                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"

+                        " attribute %s : %s", (op == GD_OP_REPLACE_BRICK) ?

+                        GF_AFR_REPLACE_BRICK : GF_AFR_ADD_BRICK,

+                        strerror (errno));

+        gf_umount_lazy (THIS->name, tmpmount, 1);

+lock:

+        synclock_lock (&priv->big_lock);

+out:

+        if (pid)

+                GF_FREE (pid);

+        gf_msg_debug ("glusterd", 0, "Returning with ret");

+        return ret;

+}

diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h

index e6380f5..6b74e90 100644

--- a/xlators/mgmt/glusterd/src/glusterd-utils.h

+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h

@@ -695,4 +695,14 @@ int

 glusterd_volume_brick_for_each (glusterd_volinfo_t *volinfo, void *data,

                int (*fn) (glusterd_volinfo_t *, glusterd_brickinfo_t *,

                           dict_t *mod_dict, void *));

+

+int

+glusterd_get_dummy_client_filepath (char *filepath,

+                                    glusterd_volinfo_t *volinfo,

+                                    gf_transport_type type);

+

+int

+glusterd_handle_replicate_brick_ops (glusterd_volinfo_t *volinfo,

+                                     glusterd_brickinfo_t *brickinfo,

+                                     glusterd_op_t op);

 #endif

diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c

index 8a86ab5..086f053 100644

--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c

+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c

@@ -5167,6 +5167,58 @@ enumerate_transport_reqs (gf_transport_type type, char **types)

 }

 int

+generate_dummy_client_volfiles (glusterd_volinfo_t *volinfo)

+{

+        int                i                  = 0;

+        int                ret                = -1;

+        char               filepath[PATH_MAX] = {0,};

+        char               *types[]           = {NULL, NULL, NULL};

+        dict_t             *dict              = NULL;

+        xlator_t           *this              = NULL;

+        gf_transport_type  type               = GF_TRANSPORT_TCP;

+

+        this = THIS;

+

+        enumerate_transport_reqs (volinfo->transport_type, types);

+        dict = dict_new ();

+        if (!dict)

+                goto out;

+        for (i = 0; types[i]; i++) {

+                memset (filepath, 0, sizeof (filepath));

+                ret = dict_set_str (dict, "client-transport-type", types[i]);

+                if (ret)

+                        goto out;

+                type = transport_str_to_type (types[i]);

+

+                ret = dict_set_uint32 (dict, "trusted-client", GF_CLIENT_OTHER);

+                if (ret)

+                        goto out;

+

+                ret = glusterd_get_dummy_client_filepath (filepath,

+                                                          volinfo, type);

+                if (ret) {

+                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,

+                                GD_MSG_INVALID_ENTRY,

+                                "Received invalid transport-type.");

+                        goto out;

+                }

+

+                ret = generate_single_transport_client_volfile (volinfo,

+                                                                filepath,

+                                                                dict);

+                if (ret)

+                        goto out;

+        }

+

+out:

+        if (dict)

+                dict_unref (dict);

+

+        gf_msg_trace ("glusterd", 0, "Returning %d", ret);

+        return ret;

+}

+

+int

 generate_client_volfiles (glusterd_volinfo_t *volinfo,

                           glusterd_client_type_t client_type)

 {

diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h

index f1dc823..c86a87b 100644

--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h

+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h

@@ -293,4 +293,7 @@ glusterd_volopt_validate (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,

 gf_boolean_t

 gd_is_self_heal_enabled (glusterd_volinfo_t *volinfo, dict_t *dict);

+int

+generate_dummy_client_volfiles (glusterd_volinfo_t *volinfo);

+

 #endif

-- 

1.7.1