|
 |
887953 |
From 43124cc2d3ddb1b3262e254326d8461cdf1f894d Mon Sep 17 00:00:00 2001
|
|
|
887953 |
From: Ravishankar N <ravishankar@redhat.com>
|
|
|
887953 |
Date: Fri, 21 Sep 2018 18:23:34 +0530
|
|
|
887953 |
Subject: [PATCH 385/385] afr: fix incorrect reporting of directory split-brain
|
|
|
887953 |
|
|
|
887953 |
Patch on master: https://review.gluster.org/#/c/glusterfs/+/21135/
|
|
|
887953 |
Problem:
|
|
|
887953 |
When a directory has dirty xattrs due to failed post-ops or when
|
|
|
887953 |
replace/reset brick is performed, AFR does a conservative merge as
|
|
|
887953 |
expected, but heal-info reports it as split-brain because there are no
|
|
|
887953 |
clear sources.
|
|
|
887953 |
|
|
|
887953 |
Fix:
|
|
|
887953 |
Modify pending flag to contain information about pending heals and
|
|
|
887953 |
split-brains. For directories, if spit-brain flag is not set,just show
|
|
|
887953 |
them as needing heal and not being in split-brain.
|
|
|
887953 |
|
|
|
887953 |
Change-Id: I21460bb5375297e421a14efb9ef0f9cea46f7e7c
|
|
|
887953 |
BUG: 1610743
|
|
|
887953 |
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
|
|
|
887953 |
Reviewed-on: https://code.engineering.redhat.com/gerrit/150742
|
|
|
887953 |
Tested-by: RHGS Build Bot <nigelb@redhat.com>
|
|
|
887953 |
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
|
|
|
887953 |
---
|
|
|
887953 |
tests/afr.rc | 2 +-
|
|
|
887953 |
.../bugs/replicate/bug-1626994-info-split-brain.t | 62 ++++++++++++++++++++++
|
|
|
887953 |
xlators/cluster/afr/src/afr-common.c | 14 ++---
|
|
|
887953 |
xlators/cluster/afr/src/afr-self-heal-common.c | 6 ++-
|
|
|
887953 |
xlators/cluster/afr/src/afr-self-heal-data.c | 2 +-
|
|
|
887953 |
xlators/cluster/afr/src/afr-self-heal-entry.c | 2 +-
|
|
|
887953 |
xlators/cluster/afr/src/afr-self-heal-metadata.c | 3 +-
|
|
|
887953 |
xlators/cluster/afr/src/afr-self-heal.h | 8 +--
|
|
|
887953 |
xlators/cluster/afr/src/afr.h | 3 ++
|
|
|
887953 |
9 files changed, 85 insertions(+), 17 deletions(-)
|
|
|
887953 |
create mode 100644 tests/bugs/replicate/bug-1626994-info-split-brain.t
|
|
|
887953 |
|
|
|
887953 |
diff --git a/tests/afr.rc b/tests/afr.rc
|
|
|
887953 |
index bdf4075..1fd0310 100644
|
|
|
887953 |
--- a/tests/afr.rc
|
|
|
887953 |
+++ b/tests/afr.rc
|
|
|
887953 |
@@ -2,7 +2,7 @@
|
|
|
887953 |
|
|
|
887953 |
function create_brick_xattrop_entry {
|
|
|
887953 |
local xattrop_dir=$(afr_get_index_path $1)
|
|
|
887953 |
- local base_entry=`ls $xattrop_dir`
|
|
|
887953 |
+ local base_entry=`ls $xattrop_dir|grep xattrop`
|
|
|
887953 |
local gfid_str
|
|
|
887953 |
local params=`echo "$@" | cut -d' ' -f2-`
|
|
|
887953 |
echo $params
|
|
|
887953 |
diff --git a/tests/bugs/replicate/bug-1626994-info-split-brain.t b/tests/bugs/replicate/bug-1626994-info-split-brain.t
|
|
|
887953 |
new file mode 100644
|
|
|
887953 |
index 0000000..86bfecb
|
|
|
887953 |
--- /dev/null
|
|
|
887953 |
+++ b/tests/bugs/replicate/bug-1626994-info-split-brain.t
|
|
|
887953 |
@@ -0,0 +1,62 @@
|
|
|
887953 |
+#!/bin/bash
|
|
|
887953 |
+
|
|
|
887953 |
+. $(dirname $0)/../../include.rc
|
|
|
887953 |
+. $(dirname $0)/../../volume.rc
|
|
|
887953 |
+. $(dirname $0)/../../afr.rc
|
|
|
887953 |
+
|
|
|
887953 |
+cleanup;
|
|
|
887953 |
+
|
|
|
887953 |
+# Test to check dirs having dirty xattr do not show up in info split-brain.
|
|
|
887953 |
+
|
|
|
887953 |
+TEST glusterd;
|
|
|
887953 |
+TEST pidof glusterd;
|
|
|
887953 |
+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2};
|
|
|
887953 |
+TEST $CLI volume set $V0 self-heal-daemon off
|
|
|
887953 |
+TEST $CLI volume start $V0;
|
|
|
887953 |
+EXPECT 'Started' volinfo_field $V0 'Status';
|
|
|
887953 |
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0
|
|
|
887953 |
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1
|
|
|
887953 |
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2
|
|
|
887953 |
+TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0
|
|
|
887953 |
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
|
|
|
887953 |
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1
|
|
|
887953 |
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2
|
|
|
887953 |
+
|
|
|
887953 |
+# Create base entry in indices/xattrop
|
|
|
887953 |
+echo "Data" > $M0/FILE
|
|
|
887953 |
+rm -f $M0/FILE
|
|
|
887953 |
+EXPECT "1" count_index_entries $B0/${V0}0
|
|
|
887953 |
+EXPECT "1" count_index_entries $B0/${V0}1
|
|
|
887953 |
+EXPECT "1" count_index_entries $B0/${V0}2
|
|
|
887953 |
+
|
|
|
887953 |
+TEST mkdir $M0/dirty_dir
|
|
|
887953 |
+TEST mkdir $M0/pending_dir
|
|
|
887953 |
+
|
|
|
887953 |
+# Set dirty xattrs on all bricks to simulate the case where entry transaction
|
|
|
887953 |
+# succeeded only the pre-op phase.
|
|
|
887953 |
+TEST setfattr -n trusted.afr.dirty -v 0x000000000000000000000001 $B0/${V0}0/dirty_dir
|
|
|
887953 |
+TEST setfattr -n trusted.afr.dirty -v 0x000000000000000000000001 $B0/${V0}1/dirty_dir
|
|
|
887953 |
+TEST setfattr -n trusted.afr.dirty -v 0x000000000000000000000001 $B0/${V0}2/dirty_dir
|
|
|
887953 |
+create_brick_xattrop_entry $B0/${V0}0 dirty_dir
|
|
|
887953 |
+# Should not show up as split-brain.
|
|
|
887953 |
+EXPECT "0" afr_get_split_brain_count $V0
|
|
|
887953 |
+
|
|
|
887953 |
+# replace/reset brick case where the new brick has dirty and the other 2 bricks
|
|
|
887953 |
+# blame it should not be reported as split-brain.
|
|
|
887953 |
+TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000000000000000000001 $B0/${V0}0
|
|
|
887953 |
+TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000000000000000000001 $B0/${V0}1
|
|
|
887953 |
+TEST setfattr -n trusted.afr.dirty -v 0x000000000000000000000001 $B0/${V0}2
|
|
|
887953 |
+create_brick_xattrop_entry $B0/${V0}0 "/"
|
|
|
887953 |
+# Should not show up as split-brain.
|
|
|
887953 |
+EXPECT "0" afr_get_split_brain_count $V0
|
|
|
887953 |
+
|
|
|
887953 |
+# Set pending xattrs on all bricks blaming each other to simulate the case of
|
|
|
887953 |
+# entry split-brain.
|
|
|
887953 |
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/${V0}0/pending_dir
|
|
|
887953 |
+TEST setfattr -n trusted.afr.$V0-client-2 -v 0x000000000000000000000001 $B0/${V0}1/pending_dir
|
|
|
887953 |
+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/${V0}2/pending_dir
|
|
|
887953 |
+create_brick_xattrop_entry $B0/${V0}0 pending_dir
|
|
|
887953 |
+# Should show up as split-brain.
|
|
|
887953 |
+EXPECT "1" afr_get_split_brain_count $V0
|
|
|
887953 |
+
|
|
|
887953 |
+cleanup;
|
|
|
887953 |
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
|
|
|
887953 |
index a85549b..bded6a2 100644
|
|
|
887953 |
--- a/xlators/cluster/afr/src/afr-common.c
|
|
|
887953 |
+++ b/xlators/cluster/afr/src/afr-common.c
|
|
|
887953 |
@@ -5472,7 +5472,7 @@ out:
|
|
|
887953 |
int
|
|
|
887953 |
afr_selfheal_locked_metadata_inspect (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
inode_t *inode, gf_boolean_t *msh,
|
|
|
887953 |
- gf_boolean_t *pending)
|
|
|
887953 |
+ unsigned char *pending)
|
|
|
887953 |
{
|
|
|
887953 |
int ret = -1;
|
|
|
887953 |
unsigned char *locked_on = NULL;
|
|
|
887953 |
@@ -5521,7 +5521,7 @@ out:
|
|
|
887953 |
int
|
|
|
887953 |
afr_selfheal_locked_data_inspect (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
fd_t *fd, gf_boolean_t *dsh,
|
|
|
887953 |
- gf_boolean_t *pflag)
|
|
|
887953 |
+ unsigned char *pflag)
|
|
|
887953 |
{
|
|
|
887953 |
int ret = -1;
|
|
|
887953 |
unsigned char *data_lock = NULL;
|
|
|
887953 |
@@ -5568,7 +5568,7 @@ out:
|
|
|
887953 |
int
|
|
|
887953 |
afr_selfheal_locked_entry_inspect (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
inode_t *inode,
|
|
|
887953 |
- gf_boolean_t *esh, gf_boolean_t *pflag)
|
|
|
887953 |
+ gf_boolean_t *esh, unsigned char *pflag)
|
|
|
887953 |
{
|
|
|
887953 |
int ret = -1;
|
|
|
887953 |
int source = -1;
|
|
|
887953 |
@@ -5619,7 +5619,7 @@ afr_selfheal_locked_entry_inspect (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
sinks, healed_sinks,
|
|
|
887953 |
locked_replies,
|
|
|
887953 |
&source, pflag);
|
|
|
887953 |
- if ((ret == 0) && source < 0)
|
|
|
887953 |
+ if ((ret == 0) && (*pflag & PFLAG_SBRAIN))
|
|
|
887953 |
ret = -EIO;
|
|
|
887953 |
*esh = afr_decide_heal_info (priv, sources, ret);
|
|
|
887953 |
}
|
|
|
887953 |
@@ -5642,7 +5642,7 @@ afr_selfheal_locked_inspect (call_frame_t *frame, xlator_t *this, uuid_t gfid,
|
|
|
887953 |
gf_boolean_t *entry_selfheal,
|
|
|
887953 |
gf_boolean_t *data_selfheal,
|
|
|
887953 |
gf_boolean_t *metadata_selfheal,
|
|
|
887953 |
- gf_boolean_t *pending)
|
|
|
887953 |
+ unsigned char *pending)
|
|
|
887953 |
|
|
|
887953 |
{
|
|
|
887953 |
int ret = -1;
|
|
|
887953 |
@@ -5730,7 +5730,7 @@ afr_get_heal_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
|
|
|
887953 |
gf_boolean_t data_selfheal = _gf_false;
|
|
|
887953 |
gf_boolean_t metadata_selfheal = _gf_false;
|
|
|
887953 |
gf_boolean_t entry_selfheal = _gf_false;
|
|
|
887953 |
- gf_boolean_t pending = _gf_false;
|
|
|
887953 |
+ unsigned char pending = 0;
|
|
|
887953 |
dict_t *dict = NULL;
|
|
|
887953 |
int ret = -1;
|
|
|
887953 |
int op_errno = 0;
|
|
|
887953 |
@@ -5750,7 +5750,7 @@ afr_get_heal_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
|
|
|
887953 |
goto out;
|
|
|
887953 |
}
|
|
|
887953 |
|
|
|
887953 |
- if (pending) {
|
|
|
887953 |
+ if (pending & PFLAG_PENDING) {
|
|
|
887953 |
size = strlen ("-pending") + 1;
|
|
|
887953 |
gf_asprintf (&substr, "-pending");
|
|
|
887953 |
if (!substr)
|
|
|
887953 |
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
|
|
|
887953 |
index 7e6a691..d04f11d 100644
|
|
|
887953 |
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
|
|
|
887953 |
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
|
|
|
887953 |
@@ -1545,7 +1545,7 @@ afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
afr_transaction_type type,
|
|
|
887953 |
unsigned char *locked_on, unsigned char *sources,
|
|
|
887953 |
unsigned char *sinks, uint64_t *witness,
|
|
|
887953 |
- gf_boolean_t *pflag)
|
|
|
887953 |
+ unsigned char *pflag)
|
|
|
887953 |
{
|
|
|
887953 |
afr_private_t *priv = NULL;
|
|
|
887953 |
int i = 0;
|
|
|
887953 |
@@ -1573,7 +1573,7 @@ afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
for (i = 0; i < priv->child_count; i++) {
|
|
|
887953 |
for (j = 0; j < priv->child_count; j++)
|
|
|
887953 |
if (matrix[i][j])
|
|
|
887953 |
- *pflag = _gf_true;
|
|
|
887953 |
+ *pflag |= PFLAG_PENDING;
|
|
|
887953 |
if (*pflag)
|
|
|
887953 |
break;
|
|
|
887953 |
}
|
|
|
887953 |
@@ -1655,6 +1655,8 @@ afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
if (locked_on[i])
|
|
|
887953 |
sinks[i] = 1;
|
|
|
887953 |
}
|
|
|
887953 |
+ if (pflag)
|
|
|
887953 |
+ *pflag |= PFLAG_SBRAIN;
|
|
|
887953 |
}
|
|
|
887953 |
|
|
|
887953 |
/* One more class of witness similar to dirty in v2 is where no pending
|
|
|
887953 |
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
|
|
|
887953 |
index 556a8f9..d3deb8f 100644
|
|
|
887953 |
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
|
|
|
887953 |
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
|
|
|
887953 |
@@ -624,7 +624,7 @@ __afr_selfheal_data_prepare (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
unsigned char *sources, unsigned char *sinks,
|
|
|
887953 |
unsigned char *healed_sinks,
|
|
|
887953 |
unsigned char *undid_pending,
|
|
|
887953 |
- struct afr_reply *replies, gf_boolean_t *pflag)
|
|
|
887953 |
+ struct afr_reply *replies, unsigned char *pflag)
|
|
|
887953 |
{
|
|
|
887953 |
int ret = -1;
|
|
|
887953 |
int source = -1;
|
|
|
887953 |
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
|
|
|
887953 |
index f6d3a8a..9f597af 100644
|
|
|
887953 |
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
|
|
|
887953 |
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
|
|
|
887953 |
@@ -496,7 +496,7 @@ __afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
unsigned char *sources, unsigned char *sinks,
|
|
|
887953 |
unsigned char *healed_sinks,
|
|
|
887953 |
struct afr_reply *replies, int *source_p,
|
|
|
887953 |
- gf_boolean_t *pflag)
|
|
|
887953 |
+ unsigned char *pflag)
|
|
|
887953 |
{
|
|
|
887953 |
int ret = -1;
|
|
|
887953 |
int source = -1;
|
|
|
887953 |
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
|
|
|
887953 |
index 199f896..50f8888 100644
|
|
|
887953 |
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
|
|
|
887953 |
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
|
|
|
887953 |
@@ -318,7 +318,8 @@ __afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this, inode_t *i
|
|
|
887953 |
unsigned char *locked_on, unsigned char *sources,
|
|
|
887953 |
unsigned char *sinks, unsigned char *healed_sinks,
|
|
|
887953 |
unsigned char *undid_pending,
|
|
|
887953 |
- struct afr_reply *replies, gf_boolean_t *pflag)
|
|
|
887953 |
+ struct afr_reply *replies,
|
|
|
887953 |
+ unsigned char *pflag)
|
|
|
887953 |
{
|
|
|
887953 |
int ret = -1;
|
|
|
887953 |
int source = -1;
|
|
|
887953 |
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
|
|
|
887953 |
index cc99d9e..7e6fb42 100644
|
|
|
887953 |
--- a/xlators/cluster/afr/src/afr-self-heal.h
|
|
|
887953 |
+++ b/xlators/cluster/afr/src/afr-self-heal.h
|
|
|
887953 |
@@ -172,7 +172,7 @@ afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
afr_transaction_type type,
|
|
|
887953 |
unsigned char *locked_on, unsigned char *sources,
|
|
|
887953 |
unsigned char *sinks, uint64_t *witness,
|
|
|
887953 |
- gf_boolean_t *flag);
|
|
|
887953 |
+ unsigned char *flag);
|
|
|
887953 |
int
|
|
|
887953 |
afr_selfheal_fill_matrix (xlator_t *this, int **matrix, int subvol, int idx,
|
|
|
887953 |
dict_t *xdata);
|
|
|
887953 |
@@ -286,7 +286,7 @@ __afr_selfheal_data_prepare (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
unsigned char *sources,
|
|
|
887953 |
unsigned char *sinks, unsigned char *healed_sinks,
|
|
|
887953 |
unsigned char *undid_pending,
|
|
|
887953 |
- struct afr_reply *replies, gf_boolean_t *flag);
|
|
|
887953 |
+ struct afr_reply *replies, unsigned char *flag);
|
|
|
887953 |
|
|
|
887953 |
int
|
|
|
887953 |
__afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
@@ -296,7 +296,7 @@ __afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
unsigned char *healed_sinks,
|
|
|
887953 |
unsigned char *undid_pending,
|
|
|
887953 |
struct afr_reply *replies,
|
|
|
887953 |
- gf_boolean_t *flag);
|
|
|
887953 |
+ unsigned char *flag);
|
|
|
887953 |
int
|
|
|
887953 |
__afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
inode_t *inode, unsigned char *locked_on,
|
|
|
887953 |
@@ -304,7 +304,7 @@ __afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
unsigned char *sinks,
|
|
|
887953 |
unsigned char *healed_sinks,
|
|
|
887953 |
struct afr_reply *replies, int *source_p,
|
|
|
887953 |
- gf_boolean_t *flag);
|
|
|
887953 |
+ unsigned char *flag);
|
|
|
887953 |
|
|
|
887953 |
int
|
|
|
887953 |
afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
|
|
|
887953 |
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
|
|
|
887953 |
index afe4a73..2e6d995 100644
|
|
|
887953 |
--- a/xlators/cluster/afr/src/afr.h
|
|
|
887953 |
+++ b/xlators/cluster/afr/src/afr.h
|
|
|
887953 |
@@ -36,6 +36,9 @@
|
|
|
887953 |
|
|
|
887953 |
#define ARBITER_BRICK_INDEX 2
|
|
|
887953 |
|
|
|
887953 |
+#define PFLAG_PENDING (1 << 0)
|
|
|
887953 |
+#define PFLAG_SBRAIN (1 << 1)
|
|
|
887953 |
+
|
|
|
887953 |
typedef int (*afr_lock_cbk_t) (call_frame_t *frame, xlator_t *this);
|
|
|
887953 |
|
|
|
887953 |
typedef int (*afr_read_txn_wind_t) (call_frame_t *frame, xlator_t *this, int subvol);
|
|
|
887953 |
--
|
|
|
887953 |
1.8.3.1
|
|
|
887953 |
|