From 4546cfb8bd7e9b42c01803a543fa52471574265d Mon Sep 17 00:00:00 2001

From: Mohit Agrawal <moagrawa@redhat.com>

Date: Fri, 19 Aug 2016 10:33:50 +0530

Subject: [PATCH 306/361] dht: "replica.split-brain-status" attribute value is

 not correct

Problem: In a distributed-replicate volume attribute

         "replica.split-brain-status" value does not display split-brain

           condition though directory is in split-brain.

         If directory is in split brain on mutiple replica-pairs

         it does not show full list of replica pairs.

Solution: Update the dht_aggregate code to aggregate the xattr

          value in this specific condition.

Fix:      1) function getChoices returns the choices from split-brain

             status string.

          2) function add_opt adding the choices to local buffer to

             store in dictionary

          3) For the key "replica.split-brain-status" function dht_aggregate

             call dht_aggregate_split_brain_xattr to prepare the list.

Test:     To verify the patch followed below steps

          1) Create a distributed replica volume and create mount point

          2) Stop heal daemon

          3) Touch file and directories on mount point

             mkdir test{1..5};touch tmp{1..5}

          4) Down brick process on one of the replica set

             pkill -9 glusterfsd

          5) Change permission of dir on mount point

             chmod 755 test{1..5}

          6) Restart brick process on node with force option

          7) kill brick process on other node in same replica set

          8) Change permission of dir again on mount point

             chmod 766 test{1..5}

          9) Reexecute same step from 4-9 on other replica set also

          10) After check heal status on server it will show dir's are

              in split brain on all replica sets

          11) After check the replica.split-brain-status attr on mount

              point it will show wrong status of split brain.

          12) After apply the patch the attribute shows correct value.

mainline:

> BUG: 1368312

> Reviewed-on: http://review.gluster.org/15201

> Smoke: Gluster Build System <jenkins@build.gluster.org>

> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

(cherry picked from commit 975932767d8131e8c331ea3eb527bd8b7240976d)

BUG: 1260779

Change-Id: Icdfd72005a4aa82337c342762775a3d1761bbe4a

Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/101282

Tested-by: Milind Changire <mchangir@redhat.com>

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

---

 tests/bugs/bug-1368312.t             |  84 ++++++++++++++

 xlators/cluster/dht/src/dht-common.c | 212 +++++++++++++++++++++++++++++++++--

 xlators/cluster/dht/src/dht-common.h |   9 ++

 3 files changed, 293 insertions(+), 12 deletions(-)

 create mode 100644 tests/bugs/bug-1368312.t

diff --git a/tests/bugs/bug-1368312.t b/tests/bugs/bug-1368312.t

new file mode 100644

index 0000000..135048f

--- /dev/null

+++ b/tests/bugs/bug-1368312.t

@@ -0,0 +1,84 @@

+#!/bin/bash

+. $(dirname $0)/../include.rc

+. $(dirname $0)/../volume.rc

+cleanup;

+

+function compare_get_split_brain_status {

+        local path=$1

+        local choice=$2

+        echo `getfattr -n replica.split-brain-status $path` | cut -f2 -d"=" | sed -e 's/^"//'  -e 's/"$//' | grep $choice

+        if [ $? -ne 0 ]

+        then

+                echo 1

+        else

+                echo 0

+        fi

+

+}

+

+TEST glusterd

+TEST pidof glusterd

+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1,2,3,4,5}

+TEST $CLI volume start $V0

+

+#Disable self-heal-daemon

+TEST $CLI volume set $V0 cluster.self-heal-daemon off

+

+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0;

+

+TEST mkdir $M0/tmp1

+

+#Create metadata split-brain

+TEST kill_brick $V0 $H0 $B0/${V0}0

+TEST chmod 666 $M0/tmp1

+TEST $CLI volume start $V0 force

+TEST kill_brick $V0 $H0 $B0/${V0}1

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0

+

+TEST chmod 757 $M0/tmp1

+

+TEST $CLI volume start $V0 force

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1

+

+EXPECT 2 get_pending_heal_count $V0

+

+

+TEST kill_brick $V0 $H0 $B0/${V0}2

+TEST chmod 755 $M0/tmp1

+TEST $CLI volume start $V0 force

+TEST kill_brick $V0 $H0 $B0/${V0}3

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2

+

+TEST chmod 766 $M0/tmp1

+

+TEST $CLI volume start $V0 force

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 3

+

+EXPECT 4 get_pending_heal_count $V0

+

+TEST kill_brick $V0 $H0 $B0/${V0}4

+TEST chmod 765 $M0/tmp1

+TEST $CLI volume start $V0 force

+TEST kill_brick $V0 $H0 $B0/${V0}5

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 4

+

+TEST chmod 756 $M0/tmp1

+

+TEST $CLI volume start $V0 force

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 4

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 5

+

+EXPECT 6 get_pending_heal_count $V0

+

+cd $M0

+EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-0

+EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-1

+EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-2

+EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-3

+EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-4

+EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-5

+

+cd -

+cleanup

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c

index 62657a0..4907931 100644

--- a/xlators/cluster/dht/src/dht-common.c

+++ b/xlators/cluster/dht/src/dht-common.c

@@ -127,6 +127,190 @@ out:

 }

+int add_opt(char **optsp, const char *opt)

+{

+        char *newopts = NULL;

+        unsigned oldsize = 0;

+        unsigned newsize = 0;

+

+        if (*optsp == NULL)

+                newopts = gf_strdup (opt);

+        else {

+                oldsize = strlen (*optsp);

+                newsize = oldsize + 1 + strlen (opt) + 1;

+                newopts = GF_REALLOC (*optsp, newsize);

+                if (newopts)

+                        sprintf (newopts + oldsize, ",%s", opt);

+        }

+        if (newopts == NULL) {

+                gf_msg ("dht", GF_LOG_WARNING, 0,

+                        DHT_MSG_NO_MEMORY,

+                        "Error to add choices in buffer in add_opt");

+                return -1;

+        }

+        *optsp = newopts;

+        return 0;

+}

+

+/* Return Choice list from Split brain status */

+char *

+getChoices (const char *value)

+{

+        int i = 0;

+        char *ptr = NULL;

+        char *tok = NULL;

+        char *result = NULL;

+        char *newval = NULL;

+

+        ptr = strstr (value, "Choices:");

+        if (!ptr) {

+                result = ptr;

+                goto out;

+        }

+

+        newval = gf_strdup (ptr);

+        if (!newval) {

+                result = newval;

+                goto out;

+        }

+

+        tok = strtok (newval, ":");

+        if (!tok) {

+                result = tok;

+                goto out;

+        }

+

+        while (tok) {

+                i++;

+                if (i == 2)

+                        break;

+                tok = strtok (NULL, ":");

+        }

+

+        result = gf_strdup (tok);

+

+out:

+        if (newval)

+                GF_FREE (newval);

+

+        return result;

+}

+

+/* This function prepare a list of choices for key

+   (replica.split-brain-status) in   case of metadata split brain

+   only on the basis of key-value passed to this function.

+   After prepare the list of choices it update the same key in dict

+   with this value to reflect the same in

+   replica.split-brain-status attr for file.

+

+*/

+

+int

+dht_aggregate_split_brain_xattr (dict_t *dst, char *key, data_t *value)

+{

+

+        int              ret            = 0;

+        char            *oldvalue       = NULL;

+        char            *old_choice     = NULL;

+        char            *new_choice     = NULL;

+        char            *full_choice    = NULL;

+        char            *status         = NULL;

+

+        if (value == NULL) {

+                gf_msg ("dht", GF_LOG_WARNING, 0,

+                        DHT_MSG_DATA_NULL,

+                        "GF_AFR_SBRAIN_STATUS value is NULL");

+                ret = -1;

+                goto out;

+        }

+

+        ret = dict_get_str (dst, key, &oldvalue);

+        if (ret)

+                goto out;

+

+        if (oldvalue && (strstr (oldvalue, "not"))) {

+                gf_msg_debug ("dht", 0,

+                              "Need to update split-brain status in dict");

+                ret = -1;

+                goto out;

+        }

+        if (oldvalue && (strstr (oldvalue, "metadata-split-brain:yes"))

+                     && (strstr (oldvalue, "data-split-brain:no"))) {

+                if (strstr (value->data, "not")) {

+                        gf_msg_debug ("dht", 0,

+                                      "No need to update split-brain status");

+                        ret = 0;

+                        goto out;

+                }

+                if (strstr (value->data, "yes") &&

+                        (strncmp (oldvalue, value->data, strlen(oldvalue)))) {

+                        old_choice = getChoices (oldvalue);

+                        if (!old_choice) {

+                                gf_msg ("dht", GF_LOG_WARNING, 0,

+                                        DHT_MSG_NO_MEMORY,

+                                        "Error to get choices");

+                                ret = -1;

+                                goto out;

+                        }

+

+                        ret = add_opt (&full_choice, old_choice);

+                        if (ret) {

+                                gf_msg ("dht", GF_LOG_WARNING, 0,

+                                         DHT_MSG_NO_MEMORY,

+                                         "Error to add choices");

+                                ret = -1;

+                                goto out;

+                        }

+

+                        new_choice = getChoices (value->data);

+                        if (!new_choice) {

+                                gf_msg ("dht", GF_LOG_WARNING, 0,

+                                        DHT_MSG_NO_MEMORY,

+                                        "Error to get choices");

+                                ret = -1;

+                                goto out;

+                        }

+

+                        ret = add_opt (&full_choice, new_choice);

+                        if (ret) {

+                                gf_msg ("dht", GF_LOG_WARNING, 0,

+                                       DHT_MSG_NO_MEMORY,

+                                       "Error to add choices ");

+                                ret = -1;

+                                goto out;

+                        }

+                        ret = gf_asprintf (&status,

+                                           "data-split-brain:%s    "

+                                           "metadata-split-brain:%s   Choices:%s",

+                                           "no", "yes", full_choice);

+

+                        if (-1 == ret) {

+                                gf_msg ("dht", GF_LOG_WARNING, 0,

+                                                 DHT_MSG_NO_MEMORY,

+                                                "Error to prepare status ");

+                                        goto out;

+                                }

+                        ret = dict_set_dynstr (dst, key, status);

+                        if (ret) {

+                                gf_msg ("dht", GF_LOG_WARNING, 0,

+                                        DHT_MSG_DICT_SET_FAILED,

+                                        "Failed to set full choice");

+                        }

+                }

+        }

+

+out:

+        if (old_choice)

+                GF_FREE (old_choice);

+        if (new_choice)

+                GF_FREE (new_choice);

+        if (full_choice)

+                GF_FREE (full_choice);

+

+        return ret;

+}

+

+

 int

 dht_aggregate (dict_t *this, char *key, data_t *value, void *data)

@@ -137,18 +321,22 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)

         dst = data;

-        if (strcmp (key, QUOTA_SIZE_KEY) == 0) {

+        /* compare split brain xattr only */

+        if (strcmp (key, GF_AFR_SBRAIN_STATUS) == 0) {

+                ret = dht_aggregate_split_brain_xattr(dst, key, value);

+                if (!ret)

+                        goto out;

+        } else if (strcmp (key, QUOTA_SIZE_KEY) == 0) {

                 ret = dht_aggregate_quota_xattr (dst, key, value);

                 if (ret) {

                         gf_msg ("dht", GF_LOG_WARNING, 0,

                                 DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,

                                 "Failed to aggregate quota xattr");

-                        goto out;

                 }

+                goto out;

         } else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {

                 ret = gf_get_min_stime (THIS, dst, key, value);

-                if (ret < 0)

-                        goto out;

+                goto out;

         } else {

                 /* compare user xattrs only */

                 if (!strncmp (key, "user.", strlen ("user."))) {

@@ -161,16 +349,16 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)

                                                       key);

                         }

                 }

-                ret = dict_set (dst, key, value);

-                if (ret) {

-                        gf_msg ("dht", GF_LOG_WARNING, 0,

-                                DHT_MSG_DICT_SET_FAILED,

-                                "Failed to set dictionary value: key = %s",

-                                key);

-                }

         }

-        ret = 0;

+        ret = dict_set (dst, key, value);

+        if (ret) {

+                gf_msg ("dht", GF_LOG_WARNING, 0,

+                        DHT_MSG_DICT_SET_FAILED,

+                        "Failed to set dictionary value: key = %s",

+                        key);

+        }

+

 out:

         return ret;

 }

diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h

index e87bd90..21e000a 100644

--- a/xlators/cluster/dht/src/dht-common.h

+++ b/xlators/cluster/dht/src/dht-common.h

@@ -1236,4 +1236,13 @@ dht_get_lock_subvolume (xlator_t *this, struct gf_flock *lock,

 int

 dht_lk_inode_unref (call_frame_t *frame, int32_t op_ret);

+int

+add_opt(char **optsp, const char *opt);

+

+char *

+getChoices (const char *value);

+

+int

+dht_aggregate_split_brain_xattr (dict_t *dst, char *key, data_t *value);

+

 #endif/* _DHT_H */

-- 

1.8.3.1