From a75bb15fbe64f14580c44b8a33314c8bbeffdede Mon Sep 17 00:00:00 2001

From: Ravishankar N <ravishankar@redhat.com>

Date: Thu, 4 Jun 2020 18:54:46 +0530

Subject: [PATCH 406/449] afr: support split-brain CLI for replica 3

Patch in upstream master: https://review.gluster.org/#/c/glusterfs/+/23502/

Ever since we added quorum checks for lookups in afr via commit

bd44d59741bb8c0f5d7a62c5b1094179dd0ce8a4, the split-brain resolution

commands would not work for replica 3 because there would be no

readables for the lookup fop.

The argument was that split-brains do not occur in replica 3 but we do

see (data/metadata) split-brain cases once in a while which indicate that there are

a few bugs/corner cases yet to be discovered and fixed.

Fortunately, commit  8016d51a3bbd410b0b927ed66be50a09574b7982 added

GF_CLIENT_PID_GLFS_HEALD as the pid for all fops made by glfsheal. If we

leverage this and allow lookups in afr when pid is GF_CLIENT_PID_GLFS_HEALD,

split-brain resolution commands will work for replica 3 volumes too.

Likewise, the check is added in shard_lookup as well to permit resolving

split-brains by specifying "/.shard/shard-file.xx" as the file name

(which previously used to fail with EPERM).

BUG: 1759875

Change-Id: I203735b909c7d30fc4faaf3ecd4f5b6b379ab266

Signed-off-by: Ravishankar N <ravishankar@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/202375

Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>

---

 .../replicate/bug-1756938-replica-3-sbrain-cli.t   | 111 +++++++++++++++++++++

 xlators/cluster/afr/src/afr-common.c               |   3 +-

 xlators/features/shard/src/shard.c                 |   3 +-

 3 files changed, 115 insertions(+), 2 deletions(-)

 create mode 100644 tests/bugs/replicate/bug-1756938-replica-3-sbrain-cli.t

diff --git a/tests/bugs/replicate/bug-1756938-replica-3-sbrain-cli.t b/tests/bugs/replicate/bug-1756938-replica-3-sbrain-cli.t

new file mode 100644

index 0000000..c1bdf34

--- /dev/null

+++ b/tests/bugs/replicate/bug-1756938-replica-3-sbrain-cli.t

@@ -0,0 +1,111 @@

+#!/bin/bash

+

+. $(dirname $0)/../../include.rc

+. $(dirname $0)/../../volume.rc

+. $(dirname $0)/../../afr.rc

+

+cleanup;

+

+TEST glusterd;

+TEST pidof glusterd;

+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2}

+TEST $CLI volume set $V0 features.shard enable

+TEST $CLI volume set $V0 features.shard-block-size 4MB

+

+TEST $CLI volume start $V0

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2

+TEST glusterfs --volfile-server=$H0 --volfile-id=/$V0 $M0

+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 0

+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1

+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 2

+

+#Create split-brain by setting afr xattrs/gfids manually.

+#file1 is non-sharded and will be in data split-brain.

+#file2 will have one shard which will be in data split-brain.

+#file3 will have one shard which will be in gfid split-brain.

+#file4 will have one shard which will be in data & metadata split-brain.

+TEST dd if=/dev/zero of=$M0/file1 bs=1024 count=1024 oflag=direct

+TEST dd if=/dev/zero of=$M0/file2 bs=1M count=6 oflag=direct

+TEST dd if=/dev/zero of=$M0/file3 bs=1M count=6 oflag=direct

+TEST dd if=/dev/zero of=$M0/file4 bs=1M count=6 oflag=direct

+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0

+

+#-------------------------------------------------------------------------------

+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000000 $B0/${V0}0/file1

+TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000000000000 $B0/${V0}0/file1

+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000000000000 $B0/${V0}1/file1

+TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000000000000 $B0/${V0}1/file1

+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000000000000 $B0/${V0}2/file1

+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000000 $B0/${V0}2/file1

+

+#-------------------------------------------------------------------------------

+gfid_f2=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/file2))

+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000000 $B0/${V0}0/.shard/$gfid_f2.1

+TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000000000000 $B0/${V0}0/.shard/$gfid_f2.1

+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000000000000 $B0/${V0}1/.shard/$gfid_f2.1

+TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000000000000 $B0/${V0}1/.shard/$gfid_f2.1

+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000000000000 $B0/${V0}2/.shard/$gfid_f2.1

+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000000 $B0/${V0}2/.shard/$gfid_f2.1

+

+#-------------------------------------------------------------------------------

+TESTS_EXPECTED_IN_LOOP=5

+function assign_new_gfid {

+    brickpath=$1

+    filename=$2

+    gfid=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $brickpath/$filename))

+    gfid_shard=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $brickpath/.shard/$gfid.1))

+

+    TEST rm $brickpath/.glusterfs/${gfid_shard:0:2}/${gfid_shard:2:2}/$gfid_shard

+    TEST setfattr -x trusted.gfid $brickpath/.shard/$gfid.1

+    new_gfid=$(get_random_gfid)

+    new_gfid_str=$(gf_gfid_xattr_to_str $new_gfid)

+    TEST setfattr -n trusted.gfid -v $new_gfid $brickpath/.shard/$gfid.1

+    TEST mkdir -p $brickpath/.glusterfs/${new_gfid_str:0:2}/${new_gfid_str:2:2}

+    TEST ln $brickpath/.shard/$gfid.1 $brickpath/.glusterfs/${new_gfid_str:0:2}/${new_gfid_str:2:2}/$new_gfid_str

+}

+assign_new_gfid $B0/$V0"1" file3

+assign_new_gfid $B0/$V0"2" file3

+

+#-------------------------------------------------------------------------------

+gfid_f4=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/file4))

+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000100000000 $B0/${V0}0/.shard/$gfid_f4.1

+TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000100000000 $B0/${V0}0/.shard/$gfid_f4.1

+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000100000000 $B0/${V0}1/.shard/$gfid_f4.1

+TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000100000000 $B0/${V0}1/.shard/$gfid_f4.1

+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000100000000 $B0/${V0}2/.shard/$gfid_f4.1

+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000100000000 $B0/${V0}2/.shard/$gfid_f4.1

+

+#-------------------------------------------------------------------------------

+#Add entry to xattrop dir on first brick and check for split-brain.

+xattrop_dir0=$(afr_get_index_path $B0/$V0"0")

+base_entry_b0=`ls $xattrop_dir0`

+

+gfid_f1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/file1))

+TEST ln  $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_f1

+

+gfid_f2_shard1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/.shard/$gfid_f2.1))

+TEST ln  $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_f2_shard1

+

+gfid_f3=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/file3))

+gfid_f3_shard1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/.shard/$gfid_f3.1))

+TEST ln $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_f3_shard1

+

+gfid_f4_shard1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/.shard/$gfid_f4.1))

+TEST ln  $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_f4_shard1

+

+#-------------------------------------------------------------------------------

+#gfid split-brain won't show up in split-brain count.

+EXPECT "3" afr_get_split_brain_count $V0

+EXPECT_NOT "^0$" get_pending_heal_count $V0

+

+#Resolve split-brains

+TEST $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}1 /file1

+GFIDSTR="gfid:$gfid_f2_shard1"

+TEST $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}1 $GFIDSTR

+TEST $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}1 /.shard/$gfid_f3.1

+TEST $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}1 /.shard/$gfid_f4.1

+TEST $CLI volume heal $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0

+cleanup;

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c

index eef7fd2..32127c6 100644

--- a/xlators/cluster/afr/src/afr-common.c

+++ b/xlators/cluster/afr/src/afr-common.c

@@ -2250,7 +2250,8 @@ afr_attempt_readsubvol_set(call_frame_t *frame, xlator_t *this,

     if ((spb_choice >= 0) &&

         (AFR_COUNT(success_replies, child_count) == child_count)) {

         *read_subvol = spb_choice;

-    } else if (!priv->quorum_count) {

+    } else if (!priv->quorum_count ||

+               frame->root->pid == GF_CLIENT_PID_GLFS_HEAL) {

         *read_subvol = afr_first_up_child(frame, this);

     } else if (priv->quorum_count &&

                afr_has_quorum(data_readable, this, NULL)) {

diff --git a/xlators/features/shard/src/shard.c b/xlators/features/shard/src/shard.c

index 2e2ef5d..16d557b 100644

--- a/xlators/features/shard/src/shard.c

+++ b/xlators/features/shard/src/shard.c

@@ -1472,7 +1472,8 @@ int shard_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc,

   shard_local_t *local = NULL;

   this->itable = loc->inode->table;

-  if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {

+  if ((frame->root->pid != GF_CLIENT_PID_GSYNCD) &&

+      (frame->root->pid != GF_CLIENT_PID_GLFS_HEAL)) {

     SHARD_ENTRY_FOP_CHECK(loc, op_errno, err);

   }

-- 

1.8.3.1