14f8ab
From 0502383024cbf7e4776816e0a992dccc484a3cf2 Mon Sep 17 00:00:00 2001
14f8ab
From: Ravishankar N <ravishankar@redhat.com>
14f8ab
Date: Tue, 8 Dec 2020 17:23:22 +0530
14f8ab
Subject: [PATCH 488/511] glusterd/afr: enable granular-entry-heal by default
14f8ab
14f8ab
XXXXXXXXXXXXXXXXXXX
14f8ab
    IMPORTANT:
14f8ab
XXXXXXXXXXXXXXXXXXXX
14f8ab
I see that for rhgs-3.5.3, GD_OP_VERSION_MAX is GD_OP_VERSION_7_0. Since
14f8ab
this patch should only act on new volumes in rhgs-3.5.4, I am bumping
14f8ab
the op-version to GD_OP_VERSION_7_1. In glusterfs upstream, the patch
14f8ab
acts only if op-version >= GD_OP_VERSION_9_0 as seen in the commit
14f8ab
messae below.
14f8ab
14f8ab
Upstream patch details:
14f8ab
/------------------------------------------------------------------------------/
14f8ab
1. The option has been enabled and tested for quite some time now in RHHI-V
14f8ab
downstream and I think it is safe to make it 'on' by default. Since it
14f8ab
is not possible to simply change it from 'off' to 'on' without breaking
14f8ab
rolling upgrades, old clients etc., I have made it default only for new volumes
14f8ab
starting from op-verison GD_OP_VERSION_9_0.
14f8ab
14f8ab
Note: If you do a volume reset, the option will be turned back off.
14f8ab
This is okay as the dir's gfid will be captured in 'xattrop' folder  and heals
14f8ab
will proceed. There might be stale entries inside entry-changes' folder,
14f8ab
which will be removed when we enable the option again.
14f8ab
14f8ab
2. I encountered a cust. issue where entry heal was pending on a dir. with
14f8ab
236436 files in it and the glustershd.log output was just stuck at
14f8ab
"performing entry selfheal", so I have added logs to give us
14f8ab
more info in DEBUG level about whether entry heal and data heal are
14f8ab
progressing (metadata heal doesn't take much time). That way, we have a
14f8ab
quick visual indication to say things are not 'stuck' if we briefly
14f8ab
enable debug logs, instead of taking statedumps or checking profile info
14f8ab
etc.
14f8ab
14f8ab
>Fixes: #1483
14f8ab
>Change-Id: I4f116f8c92f8cd33f209b758ff14f3c7e1981422
14f8ab
>Signed-off-by: Ravishankar N <ravishankar@redhat.com>
14f8ab
Upstream Patch: https://github.com/gluster/glusterfs/pull/1621
14f8ab
/------------------------------------------------------------------------------/
14f8ab
14f8ab
BUG: 1890506
14f8ab
Change-Id: If449a1e873633616cfc508d74b5c22eb434b55ae
14f8ab
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
14f8ab
Reviewed-on: https://code.engineering.redhat.com/gerrit/220555
14f8ab
Tested-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
14f8ab
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
14f8ab
---
14f8ab
 libglusterfs/src/glusterfs/globals.h               |   4 +-
14f8ab
 libglusterfs/src/syncop-utils.c                    |   4 +-
14f8ab
 tests/basic/afr/add-brick-self-heal-non-granular.t |  75 +++++++++++++
14f8ab
 tests/basic/afr/add-brick-self-heal.t              |   4 +-
14f8ab
 tests/basic/afr/bug-1130892-non-granular.t         |  77 ++++++++++++++
14f8ab
 .../basic/afr/bug-1493415-gfid-heal-non-granular.t |  79 ++++++++++++++
14f8ab
 ...507-type-mismatch-error-handling-non-granular.t | 117 +++++++++++++++++++++
14f8ab
 ...1749322-entry-heal-not-happening-non-granular.t |  90 ++++++++++++++++
14f8ab
 .../afr/replace-brick-self-heal-non-granular.t     |  65 ++++++++++++
14f8ab
 tests/basic/afr/replace-brick-self-heal.t          |   2 +-
14f8ab
 tests/bugs/replicate/bug-1130892.t                 |   2 +-
14f8ab
 tests/bugs/replicate/bug-1493415-gfid-heal.t       |   2 +-
14f8ab
 .../bug-1722507-type-mismatch-error-handling.t     |  26 +++--
14f8ab
 .../bug-1749322-entry-heal-not-happening.t         |   7 +-
14f8ab
 xlators/cluster/afr/src/afr-self-heal-common.c     |   5 +
14f8ab
 xlators/cluster/afr/src/afr-self-heal-data.c       |   3 +
14f8ab
 xlators/cluster/afr/src/afr-self-heal-entry.c      |   7 +-
14f8ab
 xlators/mgmt/glusterd/src/glusterd-utils.c         |  13 +++
14f8ab
 18 files changed, 558 insertions(+), 24 deletions(-)
14f8ab
 create mode 100644 tests/basic/afr/add-brick-self-heal-non-granular.t
14f8ab
 create mode 100644 tests/basic/afr/bug-1130892-non-granular.t
14f8ab
 create mode 100644 tests/basic/afr/bug-1493415-gfid-heal-non-granular.t
14f8ab
 create mode 100644 tests/basic/afr/bug-1722507-type-mismatch-error-handling-non-granular.t
14f8ab
 create mode 100644 tests/basic/afr/bug-1749322-entry-heal-not-happening-non-granular.t
14f8ab
 create mode 100644 tests/basic/afr/replace-brick-self-heal-non-granular.t
14f8ab
14f8ab
diff --git a/libglusterfs/src/glusterfs/globals.h b/libglusterfs/src/glusterfs/globals.h
14f8ab
index 31717ed..cc145cd 100644
14f8ab
--- a/libglusterfs/src/glusterfs/globals.h
14f8ab
+++ b/libglusterfs/src/glusterfs/globals.h
14f8ab
@@ -50,7 +50,7 @@
14f8ab
     1 /* MIN is the fresh start op-version, mostly                             \
14f8ab
          should not change */
14f8ab
 #define GD_OP_VERSION_MAX                                                      \
14f8ab
-    GD_OP_VERSION_7_0 /* MAX VERSION is the maximum                            \
14f8ab
+    GD_OP_VERSION_7_1 /* MAX VERSION is the maximum                            \
14f8ab
                          count in VME table, should                            \
14f8ab
                          keep changing with                                    \
14f8ab
                          introduction of newer                                 \
14f8ab
@@ -138,6 +138,8 @@
14f8ab
 
14f8ab
 #define GD_OP_VERSION_7_0 70000 /* Op-version for GlusterFS 7.0 */
14f8ab
 
14f8ab
+#define GD_OP_VERSION_7_1 70100 /* Op-version for GlusterFS 7.1 */
14f8ab
+
14f8ab
 #include "glusterfs/xlator.h"
14f8ab
 #include "glusterfs/options.h"
14f8ab
 
14f8ab
diff --git a/libglusterfs/src/syncop-utils.c b/libglusterfs/src/syncop-utils.c
14f8ab
index be03527..2269c76 100644
14f8ab
--- a/libglusterfs/src/syncop-utils.c
14f8ab
+++ b/libglusterfs/src/syncop-utils.c
14f8ab
@@ -495,9 +495,7 @@ syncop_dir_scan(xlator_t *subvol, loc_t *loc, int pid, void *data,
14f8ab
             if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
14f8ab
                 continue;
14f8ab
 
14f8ab
-            ret = fn(subvol, entry, loc, data);
14f8ab
-            if (ret)
14f8ab
-                break;
14f8ab
+            ret |= fn(subvol, entry, loc, data);
14f8ab
         }
14f8ab
         gf_dirent_free(&entries);
14f8ab
         if (ret)
14f8ab
diff --git a/tests/basic/afr/add-brick-self-heal-non-granular.t b/tests/basic/afr/add-brick-self-heal-non-granular.t
14f8ab
new file mode 100644
14f8ab
index 0000000..19caf24
14f8ab
--- /dev/null
14f8ab
+++ b/tests/basic/afr/add-brick-self-heal-non-granular.t
14f8ab
@@ -0,0 +1,75 @@
14f8ab
+#!/bin/bash
14f8ab
+. $(dirname $0)/../../include.rc
14f8ab
+. $(dirname $0)/../../volume.rc
14f8ab
+cleanup;
14f8ab
+
14f8ab
+TEST glusterd
14f8ab
+TEST pidof glusterd
14f8ab
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
14f8ab
+EXPECT 'Created' volinfo_field $V0 'Status';
14f8ab
+TEST $CLI volume set $V0 cluster.granular-entry-heal off
14f8ab
+TEST $CLI volume start $V0
14f8ab
+EXPECT 'Started' volinfo_field $V0 'Status';
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1
14f8ab
+
14f8ab
+TEST $CLI volume set $V0 cluster.data-self-heal off
14f8ab
+TEST $CLI volume set $V0 cluster.metadata-self-heal off
14f8ab
+TEST $CLI volume set $V0 cluster.entry-self-heal off
14f8ab
+TEST $CLI volume set $V0 cluster.heal-timeout 5
14f8ab
+
14f8ab
+TEST $CLI volume set $V0 self-heal-daemon off
14f8ab
+TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0;
14f8ab
+
14f8ab
+# Create files
14f8ab
+for i in {1..5}
14f8ab
+do
14f8ab
+        echo $i > $M0/file$i.txt
14f8ab
+done
14f8ab
+
14f8ab
+# Metadata changes
14f8ab
+TEST setfattr -n user.test -v qwerty $M0/file5.txt
14f8ab
+
14f8ab
+# Add brick1
14f8ab
+TEST $CLI volume add-brick $V0 replica 3 $H0:$B0/${V0}2
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2
14f8ab
+
14f8ab
+# New-brick should accuse the old-bricks (Simulating case for data-loss)
14f8ab
+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/${V0}2/
14f8ab
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/${V0}2/
14f8ab
+
14f8ab
+# Check if pending xattr and dirty-xattr are set for newly-added-brick
14f8ab
+EXPECT "000000000000000100000001" get_hex_xattr trusted.afr.$V0-client-2 $B0/${V0}0
14f8ab
+EXPECT "000000000000000100000001" get_hex_xattr trusted.afr.$V0-client-2 $B0/${V0}1
14f8ab
+EXPECT "000000000000000000000001" get_hex_xattr trusted.afr.dirty $B0/${V0}2
14f8ab
+
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2
14f8ab
+
14f8ab
+TEST $CLI volume set $V0 self-heal-daemon on
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 2
14f8ab
+TEST $CLI volume heal $V0
14f8ab
+
14f8ab
+# Wait for heal to complete
14f8ab
+EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0
14f8ab
+
14f8ab
+# Check if entry-heal has happened
14f8ab
+TEST diff <(ls $B0/${V0}0 | sort) <(ls $B0/${V0}2 | sort)
14f8ab
+TEST diff <(ls $B0/${V0}1 | sort) <(ls $B0/${V0}2 | sort)
14f8ab
+
14f8ab
+# Test if data was healed
14f8ab
+TEST diff $B0/${V0}0/file1.txt $B0/${V0}2/file1.txt
14f8ab
+
14f8ab
+# Test if metadata was healed and exists on both the bricks
14f8ab
+EXPECT "qwerty" get_text_xattr user.test $B0/${V0}2/file5.txt
14f8ab
+EXPECT "qwerty" get_text_xattr user.test $B0/${V0}0/file5.txt
14f8ab
+
14f8ab
+EXPECT "000000000000000000000000" get_hex_xattr trusted.afr.$V0-client-2 $B0/${V0}0
14f8ab
+EXPECT "000000000000000000000000" get_hex_xattr trusted.afr.$V0-client-2 $B0/${V0}1
14f8ab
+EXPECT "000000000000000000000000" get_hex_xattr trusted.afr.dirty $B0/${V0}2
14f8ab
+
14f8ab
+cleanup;
14f8ab
diff --git a/tests/basic/afr/add-brick-self-heal.t b/tests/basic/afr/add-brick-self-heal.t
14f8ab
index c847e22..7ebf4f6 100644
14f8ab
--- a/tests/basic/afr/add-brick-self-heal.t
14f8ab
+++ b/tests/basic/afr/add-brick-self-heal.t
14f8ab
@@ -38,8 +38,8 @@ TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/${V0
14f8ab
 TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/${V0}2/
14f8ab
 
14f8ab
 # Check if pending xattr and dirty-xattr are set for newly-added-brick
14f8ab
-EXPECT "000000000000000100000001" get_hex_xattr trusted.afr.$V0-client-2 $B0/${V0}0
14f8ab
-EXPECT "000000000000000100000001" get_hex_xattr trusted.afr.$V0-client-2 $B0/${V0}1
14f8ab
+EXPECT "000000010000000100000001" get_hex_xattr trusted.afr.$V0-client-2 $B0/${V0}0
14f8ab
+EXPECT "000000010000000100000001" get_hex_xattr trusted.afr.$V0-client-2 $B0/${V0}1
14f8ab
 EXPECT "000000000000000000000001" get_hex_xattr trusted.afr.dirty $B0/${V0}2
14f8ab
 
14f8ab
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
14f8ab
diff --git a/tests/basic/afr/bug-1130892-non-granular.t b/tests/basic/afr/bug-1130892-non-granular.t
14f8ab
new file mode 100644
14f8ab
index 0000000..3cdbc7d
14f8ab
--- /dev/null
14f8ab
+++ b/tests/basic/afr/bug-1130892-non-granular.t
14f8ab
@@ -0,0 +1,77 @@
14f8ab
+#!/bin/bash
14f8ab
+. $(dirname $0)/../../include.rc
14f8ab
+. $(dirname $0)/../../volume.rc
14f8ab
+. $(dirname $0)/../../afr.rc
14f8ab
+
14f8ab
+cleanup;
14f8ab
+
14f8ab
+TEST glusterd
14f8ab
+TEST pidof glusterd
14f8ab
+TEST $CLI volume info;
14f8ab
+
14f8ab
+# Create a 1X2 replica
14f8ab
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}-{0,1}
14f8ab
+EXPECT 'Created' volinfo_field $V0 'Status';
14f8ab
+TEST $CLI volume set $V0 cluster.granular-entry-heal off
14f8ab
+
14f8ab
+# Disable self-heal daemon
14f8ab
+TEST gluster volume set $V0 self-heal-daemon off
14f8ab
+
14f8ab
+# Enable Client side heal
14f8ab
+TEST $CLI volume set $V0 cluster.data-self-heal off
14f8ab
+TEST $CLI volume set $V0 cluster.metadata-self-heal off
14f8ab
+TEST $CLI volume set $V0 cluster.entry-self-heal off
14f8ab
+
14f8ab
+# Disable all perf-xlators
14f8ab
+TEST $CLI volume set $V0 performance.quick-read off
14f8ab
+TEST $CLI volume set $V0 performance.io-cache off
14f8ab
+TEST $CLI volume set $V0 performance.write-behind off
14f8ab
+TEST $CLI volume set $V0 performance.stat-prefetch off
14f8ab
+TEST $CLI volume set $V0 performance.read-ahead off
14f8ab
+
14f8ab
+# Volume start
14f8ab
+TEST $CLI volume start $V0;
14f8ab
+EXPECT 'Started' volinfo_field $V0 'Status';
14f8ab
+
14f8ab
+# FUSE Mount
14f8ab
+TEST ${GFS} -s $H0 --volfile-id $V0 $M0
14f8ab
+
14f8ab
+# Create files and dirs
14f8ab
+TEST mkdir -p $M0/one/two/
14f8ab
+TEST `echo "Carpe diem" > $M0/one/two/three`
14f8ab
+
14f8ab
+# Simulate disk-replacement
14f8ab
+TEST kill_brick $V0 $H0 $B0/${V0}-1
14f8ab
+EXPECT_WITHIN ${PROCESS_DOWN_TIMEOUT} "^0$" afr_child_up_status $V0 1
14f8ab
+TEST rm -rf $B0/${V0}-1/one
14f8ab
+TEST rm -rf $B0/${V0}-1/.glusterfs
14f8ab
+
14f8ab
+#Ideally, disk replacement is done using reset-brick or replace-brick gluster CLI
14f8ab
+#which will create .glusterfs folder.
14f8ab
+mkdir $B0/${V0}-1/.glusterfs && chmod 600 $B0/${V0}-1/.glusterfs
14f8ab
+
14f8ab
+# Start force
14f8ab
+TEST $CLI volume start $V0 force
14f8ab
+
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1
14f8ab
+
14f8ab
+TEST stat $M0/one
14f8ab
+
14f8ab
+sleep 1
14f8ab
+
14f8ab
+# Check pending xattrs
14f8ab
+EXPECT "00000000" afr_get_specific_changelog_xattr $B0/${V0}-0/one trusted.afr.$V0-client-1 data
14f8ab
+EXPECT_NOT "00000000" afr_get_specific_changelog_xattr $B0/${V0}-0/one trusted.afr.$V0-client-1 entry
14f8ab
+EXPECT_NOT "00000000" afr_get_specific_changelog_xattr $B0/${V0}-0/one trusted.afr.$V0-client-1 metadata
14f8ab
+
14f8ab
+TEST gluster volume set $V0 self-heal-daemon on
14f8ab
+
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1
14f8ab
+TEST $CLI volume heal $V0
14f8ab
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" is_dir_heal_done $B0/${V0}-0 $B0/${V0}-1 one
14f8ab
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" is_dir_heal_done $B0/${V0}-0 $B0/${V0}-1 one/two
14f8ab
+EXPECT_WITHIN $HEAL_TIMEOUT "Y" is_file_heal_done $B0/${V0}-0 $B0/${V0}-1 one/two/three
14f8ab
+
14f8ab
+cleanup;
14f8ab
diff --git a/tests/basic/afr/bug-1493415-gfid-heal-non-granular.t b/tests/basic/afr/bug-1493415-gfid-heal-non-granular.t
14f8ab
new file mode 100644
14f8ab
index 0000000..aff001c
14f8ab
--- /dev/null
14f8ab
+++ b/tests/basic/afr/bug-1493415-gfid-heal-non-granular.t
14f8ab
@@ -0,0 +1,79 @@
14f8ab
+#!/bin/bash
14f8ab
+. $(dirname $0)/../../include.rc
14f8ab
+. $(dirname $0)/../../volume.rc
14f8ab
+. $(dirname $0)/../../afr.rc
14f8ab
+cleanup;
14f8ab
+
14f8ab
+TEST glusterd
14f8ab
+TEST pidof glusterd
14f8ab
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
14f8ab
+TEST $CLI volume set $V0 cluster.granular-entry-heal off
14f8ab
+TEST $CLI volume start $V0
14f8ab
+
14f8ab
+TEST $GFS --volfile-id=$V0 --volfile-server=$H0 --attribute-timeout=0 --entry-timeout=0 $M0;
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 0
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1
14f8ab
+TEST $CLI volume set $V0 self-heal-daemon off
14f8ab
+
14f8ab
+# Create base entry in indices/xattrop
14f8ab
+echo "Data" > $M0/FILE
14f8ab
+
14f8ab
+#------------------------------------------------------------------------------#
14f8ab
+TEST touch $M0/f1
14f8ab
+gfid_f1=$(gf_get_gfid_xattr $B0/${V0}0/f1)
14f8ab
+gfid_str_f1=$(gf_gfid_xattr_to_str $gfid_f1)
14f8ab
+
14f8ab
+# Remove gfid xattr and .glusterfs hard link from 2nd brick. This simulates a
14f8ab
+# brick crash at the point where file got created but no xattrs were set.
14f8ab
+TEST setfattr -x trusted.gfid $B0/${V0}1/f1
14f8ab
+TEST rm $B0/${V0}1/.glusterfs/${gfid_str_f1:0:2}/${gfid_str_f1:2:2}/$gfid_str_f1
14f8ab
+
14f8ab
+# storage/posix considers that a file without gfid changed less than a second
14f8ab
+# before doesn't exist, so we need to wait for a second to force posix to
14f8ab
+# consider that this is a valid file but without gfid.
14f8ab
+sleep 2
14f8ab
+
14f8ab
+# Assume there were no pending xattrs on parent dir due to 1st brick crashing
14f8ab
+# too. Then name heal from client must heal the gfid.
14f8ab
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
14f8ab
+TEST $GFS --volfile-id=$V0 --volfile-server=$H0 --attribute-timeout=0 --entry-timeout=0 $M0;
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 0
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1
14f8ab
+TEST stat $M0/f1
14f8ab
+EXPECT "$gfid_f1" gf_get_gfid_xattr $B0/${V0}1/f1
14f8ab
+TEST stat $B0/${V0}1/.glusterfs/${gfid_str_f1:0:2}/${gfid_str_f1:2:2}/$gfid_str_f1
14f8ab
+
14f8ab
+#------------------------------------------------------------------------------#
14f8ab
+TEST mkdir $M0/dir
14f8ab
+TEST touch $M0/dir/f2
14f8ab
+gfid_f2=$(gf_get_gfid_xattr $B0/${V0}0/dir/f2)
14f8ab
+gfid_str_f2=$(gf_gfid_xattr_to_str $gfid_f2)
14f8ab
+
14f8ab
+# Remove gfid xattr and .glusterfs hard link from 2nd brick. This simulates a
14f8ab
+# brick crash at the point where file got created but no xattrs were set.
14f8ab
+TEST setfattr -x trusted.gfid $B0/${V0}1/dir/f2
14f8ab
+TEST rm $B0/${V0}1/.glusterfs/${gfid_str_f2:0:2}/${gfid_str_f2:2:2}/$gfid_str_f2
14f8ab
+
14f8ab
+#Now simulate setting of pending entry xattr on parent dir of 1st brick.
14f8ab
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/${V0}0/dir
14f8ab
+create_brick_xattrop_entry $B0/${V0}0 dir
14f8ab
+
14f8ab
+# storage/posix considers that a file without gfid changed less than a second
14f8ab
+# before doesn't exist, so we need to wait for a second to force posix to
14f8ab
+# consider that this is a valid file but without gfid.
14f8ab
+sleep 2
14f8ab
+
14f8ab
+#Trigger entry-heal via shd
14f8ab
+TEST $CLI volume set $V0 self-heal-daemon on
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1
14f8ab
+
14f8ab
+TEST $CLI volume heal $V0
14f8ab
+EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0
14f8ab
+
14f8ab
+EXPECT "$gfid_f2" gf_get_gfid_xattr $B0/${V0}1/dir/f2
14f8ab
+TEST stat $B0/${V0}1/.glusterfs/${gfid_str_f2:0:2}/${gfid_str_f2:2:2}/$gfid_str_f2
14f8ab
+
14f8ab
+#------------------------------------------------------------------------------#
14f8ab
+cleanup;
14f8ab
diff --git a/tests/basic/afr/bug-1722507-type-mismatch-error-handling-non-granular.t b/tests/basic/afr/bug-1722507-type-mismatch-error-handling-non-granular.t
14f8ab
new file mode 100644
14f8ab
index 0000000..9079c93
14f8ab
--- /dev/null
14f8ab
+++ b/tests/basic/afr/bug-1722507-type-mismatch-error-handling-non-granular.t
14f8ab
@@ -0,0 +1,117 @@
14f8ab
+#!/bin/bash
14f8ab
+
14f8ab
+. $(dirname $0)/../../include.rc
14f8ab
+. $(dirname $0)/../../volume.rc
14f8ab
+. $(dirname $0)/../../afr.rc
14f8ab
+
14f8ab
+cleanup;
14f8ab
+
14f8ab
+## Start and create a volume
14f8ab
+TEST glusterd;
14f8ab
+TEST pidof glusterd;
14f8ab
+TEST $CLI volume info;
14f8ab
+
14f8ab
+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2};
14f8ab
+TEST $CLI volume set $V0 cluster.granular-entry-heal off
14f8ab
+TEST $CLI volume start $V0;
14f8ab
+TEST $CLI volume set $V0 cluster.heal-timeout 5
14f8ab
+TEST $CLI volume heal $V0 disable
14f8ab
+EXPECT 'Started' volinfo_field $V0 'Status';
14f8ab
+TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0
14f8ab
+
14f8ab
+TEST mkdir $M0/dir
14f8ab
+
14f8ab
+##########################################################################################
14f8ab
+# GFID link file and the GFID is missing on one brick and all the bricks are being blamed.
14f8ab
+
14f8ab
+TEST touch $M0/dir/file
14f8ab
+TEST `echo append>> $M0/dir/file`
14f8ab
+
14f8ab
+#B0 and B2 must blame B1
14f8ab
+setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/$V0"2"/dir
14f8ab
+setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/$V0"0"/dir
14f8ab
+setfattr -n trusted.afr.$V0-client-2 -v 0x000000000000000000000001 $B0/$V0"0"/dir
14f8ab
+
14f8ab
+# Add entry to xattrop dir to trigger index heal.
14f8ab
+xattrop_dir0=$(afr_get_index_path $B0/$V0"0")
14f8ab
+base_entry_b0=`ls $xattrop_dir0`
14f8ab
+gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/dir/))
14f8ab
+ln $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_str
14f8ab
+EXPECT "^1$" get_pending_heal_count $V0
14f8ab
+
14f8ab
+# Remove the gfid xattr and the link file on one brick.
14f8ab
+gfid_file=$(gf_get_gfid_xattr $B0/$V0"0"/dir/file)
14f8ab
+gfid_str_file=$(gf_gfid_xattr_to_str $gfid_file)
14f8ab
+TEST setfattr -x trusted.gfid $B0/${V0}0/dir/file
14f8ab
+TEST rm -f $B0/${V0}0/.glusterfs/${gfid_str_file:0:2}/${gfid_str_file:2:2}/$gfid_str_file
14f8ab
+
14f8ab
+# Launch heal
14f8ab
+TEST $CLI volume heal $V0 enable
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^Y$" glustershd_up_status
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status_in_shd $V0 0
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status_in_shd $V0 1
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status_in_shd $V0 2
14f8ab
+
14f8ab
+# Wait for 2 second to force posix to consider that this is a valid file but
14f8ab
+# without gfid.
14f8ab
+sleep 2
14f8ab
+TEST $CLI volume heal $V0
14f8ab
+
14f8ab
+# Heal should not fail as the file is missing gfid xattr and the link file,
14f8ab
+# which is not actually the gfid or type mismatch.
14f8ab
+EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0
14f8ab
+
14f8ab
+EXPECT "$gfid_file" gf_get_gfid_xattr $B0/${V0}0/dir/file
14f8ab
+TEST stat $B0/${V0}0/.glusterfs/${gfid_str_file:0:2}/${gfid_str_file:2:2}/$gfid_str_file
14f8ab
+rm -f $M0/dir/file
14f8ab
+
14f8ab
+
14f8ab
+###########################################################################################
14f8ab
+# GFID link file and the GFID is missing on two bricks and all the bricks are being blamed.
14f8ab
+
14f8ab
+TEST $CLI volume heal $V0 disable
14f8ab
+TEST touch $M0/dir/file
14f8ab
+#TEST kill_brick $V0 $H0 $B0/$V0"1"
14f8ab
+
14f8ab
+#B0 and B2 must blame B1
14f8ab
+setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/$V0"2"/dir
14f8ab
+setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/$V0"0"/dir
14f8ab
+setfattr -n trusted.afr.$V0-client-2 -v 0x000000000000000000000001 $B0/$V0"0"/dir
14f8ab
+
14f8ab
+# Add entry to xattrop dir to trigger index heal.
14f8ab
+xattrop_dir0=$(afr_get_index_path $B0/$V0"0")
14f8ab
+base_entry_b0=`ls $xattrop_dir0`
14f8ab
+gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/dir/))
14f8ab
+ln $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_str
14f8ab
+EXPECT "^1$" get_pending_heal_count $V0
14f8ab
+
14f8ab
+# Remove the gfid xattr and the link file on two bricks.
14f8ab
+gfid_file=$(gf_get_gfid_xattr $B0/$V0"0"/dir/file)
14f8ab
+gfid_str_file=$(gf_gfid_xattr_to_str $gfid_file)
14f8ab
+TEST setfattr -x trusted.gfid $B0/${V0}0/dir/file
14f8ab
+TEST rm -f $B0/${V0}0/.glusterfs/${gfid_str_file:0:2}/${gfid_str_file:2:2}/$gfid_str_file
14f8ab
+TEST setfattr -x trusted.gfid $B0/${V0}1/dir/file
14f8ab
+TEST rm -f $B0/${V0}1/.glusterfs/${gfid_str_file:0:2}/${gfid_str_file:2:2}/$gfid_str_file
14f8ab
+
14f8ab
+# Launch heal
14f8ab
+TEST $CLI volume heal $V0 enable
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^Y$" glustershd_up_status
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status_in_shd $V0 0
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status_in_shd $V0 1
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status_in_shd $V0 2
14f8ab
+
14f8ab
+# Wait for 2 second to force posix to consider that this is a valid file but
14f8ab
+# without gfid.
14f8ab
+sleep 2
14f8ab
+TEST $CLI volume heal $V0
14f8ab
+
14f8ab
+# Heal should not fail as the file is missing gfid xattr and the link file,
14f8ab
+# which is not actually the gfid or type mismatch.
14f8ab
+EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0
14f8ab
+
14f8ab
+EXPECT "$gfid_file" gf_get_gfid_xattr $B0/${V0}0/dir/file
14f8ab
+TEST stat $B0/${V0}0/.glusterfs/${gfid_str_file:0:2}/${gfid_str_file:2:2}/$gfid_str_file
14f8ab
+EXPECT "$gfid_file" gf_get_gfid_xattr $B0/${V0}1/dir/file
14f8ab
+TEST stat $B0/${V0}1/.glusterfs/${gfid_str_file:0:2}/${gfid_str_file:2:2}/$gfid_str_file
14f8ab
+
14f8ab
+cleanup
14f8ab
diff --git a/tests/basic/afr/bug-1749322-entry-heal-not-happening-non-granular.t b/tests/basic/afr/bug-1749322-entry-heal-not-happening-non-granular.t
14f8ab
new file mode 100644
14f8ab
index 0000000..4f27da4
14f8ab
--- /dev/null
14f8ab
+++ b/tests/basic/afr/bug-1749322-entry-heal-not-happening-non-granular.t
14f8ab
@@ -0,0 +1,90 @@
14f8ab
+#!/bin/bash
14f8ab
+
14f8ab
+. $(dirname $0)/../../include.rc
14f8ab
+. $(dirname $0)/../../volume.rc
14f8ab
+. $(dirname $0)/../../afr.rc
14f8ab
+
14f8ab
+cleanup
14f8ab
+
14f8ab
+function check_gfid_and_link_count
14f8ab
+{
14f8ab
+        local file=$1
14f8ab
+
14f8ab
+        file_gfid_b0=$(gf_get_gfid_xattr $B0/${V0}0/$file)
14f8ab
+        TEST [ ! -z $file_gfid_b0 ]
14f8ab
+        file_gfid_b1=$(gf_get_gfid_xattr $B0/${V0}1/$file)
14f8ab
+        file_gfid_b2=$(gf_get_gfid_xattr $B0/${V0}2/$file)
14f8ab
+        EXPECT $file_gfid_b0 echo $file_gfid_b1
14f8ab
+        EXPECT $file_gfid_b0 echo $file_gfid_b2
14f8ab
+
14f8ab
+        EXPECT "2" stat -c %h $B0/${V0}0/$file
14f8ab
+        EXPECT "2" stat -c %h $B0/${V0}1/$file
14f8ab
+        EXPECT "2" stat -c %h $B0/${V0}2/$file
14f8ab
+}
14f8ab
+TESTS_EXPECTED_IN_LOOP=18
14f8ab
+
14f8ab
+################################################################################
14f8ab
+## Start and create a volume
14f8ab
+TEST glusterd;
14f8ab
+TEST pidof glusterd;
14f8ab
+TEST $CLI volume info;
14f8ab
+
14f8ab
+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2};
14f8ab
+TEST $CLI volume set $V0 cluster.granular-entry-heal off
14f8ab
+TEST $CLI volume start $V0;
14f8ab
+TEST $CLI volume set $V0 cluster.heal-timeout 5
14f8ab
+TEST $CLI volume heal $V0 disable
14f8ab
+EXPECT 'Started' volinfo_field $V0 'Status';
14f8ab
+TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0
14f8ab
+
14f8ab
+TEST mkdir $M0/dir
14f8ab
+TEST `echo "File 1 " > $M0/dir/file1`
14f8ab
+TEST touch $M0/dir/file{2..4}
14f8ab
+
14f8ab
+# Remove file2 from 1st & 3rd bricks
14f8ab
+TEST rm -f $B0/$V0"0"/dir/file2
14f8ab
+TEST rm -f $B0/$V0"2"/dir/file2
14f8ab
+
14f8ab
+# Remove file3 and the .glusterfs hardlink from 1st & 2nd bricks
14f8ab
+gfid_file3=$(gf_get_gfid_xattr $B0/$V0"0"/dir/file3)
14f8ab
+gfid_str_file3=$(gf_gfid_xattr_to_str $gfid_file3)
14f8ab
+TEST rm $B0/$V0"0"/.glusterfs/${gfid_str_file3:0:2}/${gfid_str_file3:2:2}/$gfid_str_file3
14f8ab
+TEST rm $B0/$V0"1"/.glusterfs/${gfid_str_file3:0:2}/${gfid_str_file3:2:2}/$gfid_str_file3
14f8ab
+TEST rm -f $B0/$V0"0"/dir/file3
14f8ab
+TEST rm -f $B0/$V0"1"/dir/file3
14f8ab
+
14f8ab
+# Remove the .glusterfs hardlink and the gfid xattr of file4 on 3rd brick
14f8ab
+gfid_file4=$(gf_get_gfid_xattr $B0/$V0"0"/dir/file4)
14f8ab
+gfid_str_file4=$(gf_gfid_xattr_to_str $gfid_file4)
14f8ab
+TEST rm $B0/$V0"2"/.glusterfs/${gfid_str_file4:0:2}/${gfid_str_file4:2:2}/$gfid_str_file4
14f8ab
+TEST setfattr -x trusted.gfid $B0/$V0"2"/dir/file4
14f8ab
+
14f8ab
+# B0 and B2 blame each other
14f8ab
+setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/$V0"2"/dir
14f8ab
+setfattr -n trusted.afr.$V0-client-2 -v 0x000000000000000000000001 $B0/$V0"0"/dir
14f8ab
+
14f8ab
+# Add entry to xattrop dir on first brick.
14f8ab
+xattrop_dir0=$(afr_get_index_path $B0/$V0"0")
14f8ab
+base_entry_b0=`ls $xattrop_dir0`
14f8ab
+gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/dir/))
14f8ab
+TEST ln $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_str
14f8ab
+
14f8ab
+EXPECT "^1$" get_pending_heal_count $V0
14f8ab
+
14f8ab
+# Launch heal
14f8ab
+TEST $CLI volume heal $V0 enable
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^Y$" glustershd_up_status
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status_in_shd $V0 0
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status_in_shd $V0 1
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status_in_shd $V0 2
14f8ab
+TEST $CLI volume heal $V0
14f8ab
+EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0
14f8ab
+
14f8ab
+# All the files must be present on all the bricks after conservative merge and
14f8ab
+# should have the gfid xattr and the .glusterfs hardlink.
14f8ab
+check_gfid_and_link_count dir/file1
14f8ab
+check_gfid_and_link_count dir/file2
14f8ab
+check_gfid_and_link_count dir/file3
14f8ab
+check_gfid_and_link_count dir/file4
14f8ab
+
14f8ab
+cleanup
14f8ab
diff --git a/tests/basic/afr/replace-brick-self-heal-non-granular.t b/tests/basic/afr/replace-brick-self-heal-non-granular.t
14f8ab
new file mode 100644
14f8ab
index 0000000..c86bff1
14f8ab
--- /dev/null
14f8ab
+++ b/tests/basic/afr/replace-brick-self-heal-non-granular.t
14f8ab
@@ -0,0 +1,65 @@
14f8ab
+#!/bin/bash
14f8ab
+. $(dirname $0)/../../include.rc
14f8ab
+. $(dirname $0)/../../volume.rc
14f8ab
+cleanup;
14f8ab
+
14f8ab
+TEST glusterd
14f8ab
+TEST pidof glusterd
14f8ab
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
14f8ab
+TEST $CLI volume set $V0 cluster.granular-entry-heal off
14f8ab
+TEST $CLI volume start $V0
14f8ab
+TEST $CLI volume set $V0 cluster.data-self-heal off
14f8ab
+TEST $CLI volume set $V0 cluster.metadata-self-heal off
14f8ab
+TEST $CLI volume set $V0 cluster.entry-self-heal off
14f8ab
+TEST $CLI volume set $V0 cluster.heal-timeout 5
14f8ab
+TEST $CLI volume set $V0 self-heal-daemon off
14f8ab
+TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0;
14f8ab
+
14f8ab
+# Create files
14f8ab
+for i in {1..5}
14f8ab
+do
14f8ab
+        echo $i > $M0/file$i.txt
14f8ab
+done
14f8ab
+
14f8ab
+# Metadata changes
14f8ab
+TEST setfattr -n user.test -v qwerty $M0/file5.txt
14f8ab
+
14f8ab
+# Replace brick1
14f8ab
+TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}1_new commit force
14f8ab
+
14f8ab
+# Replaced-brick should accuse the non-replaced-brick (Simulating case for data-loss)
14f8ab
+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/${V0}1_new/
14f8ab
+
14f8ab
+# Check if pending xattr and dirty-xattr are set for replaced-brick
14f8ab
+EXPECT "000000000000000100000001" get_hex_xattr trusted.afr.$V0-client-1 $B0/${V0}0
14f8ab
+EXPECT "000000000000000000000001" get_hex_xattr trusted.afr.dirty $B0/${V0}1_new
14f8ab
+
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1
14f8ab
+
14f8ab
+TEST $CLI volume set $V0 self-heal-daemon on
14f8ab
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0
14f8ab
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1
14f8ab
+TEST $CLI volume heal $V0
14f8ab
+
14f8ab
+# Wait for heal to complete
14f8ab
+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0
14f8ab
+
14f8ab
+# Check if entry-heal has happened
14f8ab
+TEST diff <(ls $B0/${V0}0 | sort) <(ls $B0/${V0}1_new | sort)
14f8ab
+
14f8ab
+# To make sure that files were not lost from brick0
14f8ab
+TEST diff <(ls $B0/${V0}0 | sort) <(ls $B0/${V0}1 | sort)
14f8ab
+EXPECT "000000000000000000000000" get_hex_xattr trusted.afr.$V0-client-1 $B0/${V0}0
14f8ab
+
14f8ab
+# Test if data was healed
14f8ab
+TEST diff $B0/${V0}0/file1.txt $B0/${V0}1_new/file1.txt
14f8ab
+# To make sure that data was not lost from brick0
14f8ab
+TEST diff $B0/${V0}0/file1.txt $B0/${V0}1/file1.txt
14f8ab
+
14f8ab
+# Test if metadata was healed and exists on both the bricks
14f8ab
+EXPECT "qwerty" get_text_xattr user.test $B0/${V0}1_new/file5.txt
14f8ab
+EXPECT "qwerty" get_text_xattr user.test $B0/${V0}0/file5.txt
14f8ab
+
14f8ab
+cleanup;
14f8ab
diff --git a/tests/basic/afr/replace-brick-self-heal.t b/tests/basic/afr/replace-brick-self-heal.t
14f8ab
index 0360db7..da31c87 100644
14f8ab
--- a/tests/basic/afr/replace-brick-self-heal.t
14f8ab
+++ b/tests/basic/afr/replace-brick-self-heal.t
14f8ab
@@ -30,7 +30,7 @@ TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}1_new commit forc
14f8ab
 TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/${V0}1_new/
14f8ab
 
14f8ab
 # Check if pending xattr and dirty-xattr are set for replaced-brick
14f8ab
-EXPECT "000000000000000100000001" get_hex_xattr trusted.afr.$V0-client-1 $B0/${V0}0
14f8ab
+EXPECT "000000010000000100000001" get_hex_xattr trusted.afr.$V0-client-1 $B0/${V0}0
14f8ab
 EXPECT "000000000000000000000001" get_hex_xattr trusted.afr.dirty $B0/${V0}1_new
14f8ab
 
14f8ab
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
14f8ab
diff --git a/tests/bugs/replicate/bug-1130892.t b/tests/bugs/replicate/bug-1130892.t
14f8ab
index 0f57d66..e23eb26 100644
14f8ab
--- a/tests/bugs/replicate/bug-1130892.t
14f8ab
+++ b/tests/bugs/replicate/bug-1130892.t
14f8ab
@@ -56,7 +56,7 @@ EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1
14f8ab
 TEST stat $M0/one
14f8ab
 
14f8ab
 # Check pending xattrs
14f8ab
-EXPECT "00000000" afr_get_specific_changelog_xattr $B0/${V0}-0/one trusted.afr.$V0-client-1 data
14f8ab
+EXPECT "00000001" afr_get_specific_changelog_xattr $B0/${V0}-0/one trusted.afr.$V0-client-1 data
14f8ab
 EXPECT_NOT "00000000" afr_get_specific_changelog_xattr $B0/${V0}-0/one trusted.afr.$V0-client-1 entry
14f8ab
 EXPECT "00000000" afr_get_specific_changelog_xattr $B0/${V0}-0/one trusted.afr.$V0-client-1 metadata
14f8ab
 
14f8ab
diff --git a/tests/bugs/replicate/bug-1493415-gfid-heal.t b/tests/bugs/replicate/bug-1493415-gfid-heal.t
14f8ab
index 125c35a..9714d5e 100644
14f8ab
--- a/tests/bugs/replicate/bug-1493415-gfid-heal.t
14f8ab
+++ b/tests/bugs/replicate/bug-1493415-gfid-heal.t
14f8ab
@@ -49,7 +49,7 @@ TEST setfattr -x trusted.gfid $B0/${V0}1/dir/f2
14f8ab
 TEST rm $B0/${V0}1/.glusterfs/${gfid_str_f2:0:2}/${gfid_str_f2:2:2}/$gfid_str_f2
14f8ab
 
14f8ab
 #Now simulate setting of pending entry xattr on parent dir of 1st brick.
14f8ab
-TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/${V0}0/dir
14f8ab
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000001 $B0/${V0}0/dir
14f8ab
 create_brick_xattrop_entry $B0/${V0}0 dir
14f8ab
 
14f8ab
 #Trigger entry-heal via shd
14f8ab
diff --git a/tests/bugs/replicate/bug-1722507-type-mismatch-error-handling.t b/tests/bugs/replicate/bug-1722507-type-mismatch-error-handling.t
14f8ab
index 0aeaaaf..1fdf7ea 100644
14f8ab
--- a/tests/bugs/replicate/bug-1722507-type-mismatch-error-handling.t
14f8ab
+++ b/tests/bugs/replicate/bug-1722507-type-mismatch-error-handling.t
14f8ab
@@ -23,19 +23,21 @@ TEST mkdir $M0/dir
14f8ab
 ##########################################################################################
14f8ab
 # GFID link file and the GFID is missing on one brick and all the bricks are being blamed.
14f8ab
 
14f8ab
-TEST touch $M0/dir/file
14f8ab
-#TEST kill_brick $V0 $H0 $B0/$V0"1"
14f8ab
+TEST `echo append>> $M0/dir/file`
14f8ab
 
14f8ab
 #B0 and B2 must blame B1
14f8ab
-setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/$V0"2"/dir
14f8ab
-setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/$V0"0"/dir
14f8ab
-setfattr -n trusted.afr.$V0-client-2 -v 0x000000000000000000000001 $B0/$V0"0"/dir
14f8ab
+# Set data part of the xattr also to 1 so that local->need_full_crawl is true.
14f8ab
+# Another way is to create the needed entries inside indices/entry-changes
14f8ab
+# folder.
14f8ab
+setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000000000001 $B0/$V0"2"/dir
14f8ab
+setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000001 $B0/$V0"0"/dir
14f8ab
+setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000000000001 $B0/$V0"0"/dir
14f8ab
 
14f8ab
 # Add entry to xattrop dir to trigger index heal.
14f8ab
 xattrop_dir0=$(afr_get_index_path $B0/$V0"0")
14f8ab
 base_entry_b0=`ls $xattrop_dir0`
14f8ab
 gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/dir/))
14f8ab
-ln -s $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_str
14f8ab
+ln $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_str
14f8ab
 EXPECT "^1$" get_pending_heal_count $V0
14f8ab
 
14f8ab
 # Remove the gfid xattr and the link file on one brick.
14f8ab
@@ -70,18 +72,20 @@ rm -f $M0/dir/file
14f8ab
 
14f8ab
 TEST $CLI volume heal $V0 disable
14f8ab
 TEST touch $M0/dir/file
14f8ab
-#TEST kill_brick $V0 $H0 $B0/$V0"1"
14f8ab
 
14f8ab
 #B0 and B2 must blame B1
14f8ab
-setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/$V0"2"/dir
14f8ab
-setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/$V0"0"/dir
14f8ab
-setfattr -n trusted.afr.$V0-client-2 -v 0x000000000000000000000001 $B0/$V0"0"/dir
14f8ab
+# Set data part of the xattr also to 1 so that local->need_full_crawl is true.
14f8ab
+# Another way is to create the needed entries inside indices/entry-changes
14f8ab
+# folder.
14f8ab
+setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000000000001 $B0/$V0"2"/dir
14f8ab
+setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000001 $B0/$V0"0"/dir
14f8ab
+setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000000000001 $B0/$V0"0"/dir
14f8ab
 
14f8ab
 # Add entry to xattrop dir to trigger index heal.
14f8ab
 xattrop_dir0=$(afr_get_index_path $B0/$V0"0")
14f8ab
 base_entry_b0=`ls $xattrop_dir0`
14f8ab
 gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/dir/))
14f8ab
-ln -s $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_str
14f8ab
+ln $xattrop_dir0/$base_entry_b0 $xattrop_dir0/$gfid_str
14f8ab
 EXPECT "^1$" get_pending_heal_count $V0
14f8ab
 
14f8ab
 # Remove the gfid xattr and the link file on two bricks.
14f8ab
diff --git a/tests/bugs/replicate/bug-1749322-entry-heal-not-happening.t b/tests/bugs/replicate/bug-1749322-entry-heal-not-happening.t
14f8ab
index 9627908..3da873a 100644
14f8ab
--- a/tests/bugs/replicate/bug-1749322-entry-heal-not-happening.t
14f8ab
+++ b/tests/bugs/replicate/bug-1749322-entry-heal-not-happening.t
14f8ab
@@ -59,8 +59,11 @@ TEST rm $B0/$V0"2"/.glusterfs/${gfid_str_file4:0:2}/${gfid_str_file4:2:2}/$gfid_
14f8ab
 TEST setfattr -x trusted.gfid $B0/$V0"2"/dir/file4
14f8ab
 
14f8ab
 # B0 and B2 blame each other
14f8ab
-setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/$V0"2"/dir
14f8ab
-setfattr -n trusted.afr.$V0-client-2 -v 0x000000000000000000000001 $B0/$V0"0"/dir
14f8ab
+# Set data part of the xattr also to 1 so that local->need_full_crawl is true.
14f8ab
+# Another way is to create the needed entries inside indices/entry-changes
14f8ab
+# folder.
14f8ab
+setfattr -n trusted.afr.$V0-client-0 -v 0x000000010000000000000001 $B0/$V0"2"/dir
14f8ab
+setfattr -n trusted.afr.$V0-client-2 -v 0x000000010000000000000001 $B0/$V0"0"/dir
14f8ab
 
14f8ab
 # Add entry to xattrop dir on first brick.
14f8ab
 xattrop_dir0=$(afr_get_index_path $B0/$V0"0")
14f8ab
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
14f8ab
index 1608f75..36fd3a9 100644
14f8ab
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
14f8ab
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
14f8ab
@@ -2549,6 +2549,11 @@ afr_selfheal_do(call_frame_t *frame, xlator_t *this, uuid_t gfid)
14f8ab
         }
14f8ab
     }
14f8ab
 
14f8ab
+    gf_msg_debug(
14f8ab
+        this->name, 0,
14f8ab
+        "heals needed for %s: [entry-heal=%d, metadata-heal=%d, data-heal=%d]",
14f8ab
+        uuid_utoa(gfid), entry_selfheal, metadata_selfheal, data_selfheal);
14f8ab
+
14f8ab
     if (data_selfheal && priv->data_self_heal)
14f8ab
         data_ret = afr_selfheal_data(frame, this, fd);
14f8ab
 
14f8ab
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
14f8ab
index cdff4a5..b97c66b 100644
14f8ab
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
14f8ab
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
14f8ab
@@ -239,6 +239,9 @@ afr_selfheal_data_block(call_frame_t *frame, xlator_t *this, fd_t *fd,
14f8ab
     sink_count = AFR_COUNT(healed_sinks, priv->child_count);
14f8ab
     data_lock = alloca0(priv->child_count);
14f8ab
 
14f8ab
+    gf_msg_debug(this->name, 0, "gfid:%s, offset=%jd, size=%zu",
14f8ab
+                 uuid_utoa(fd->inode->gfid), offset, size);
14f8ab
+
14f8ab
     ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, offset, size,
14f8ab
                                data_lock);
14f8ab
     {
14f8ab
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
14f8ab
index 40be898..00b5b2d 100644
14f8ab
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
14f8ab
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
14f8ab
@@ -206,8 +206,11 @@ __afr_selfheal_heal_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
14f8ab
                                             replies);
14f8ab
         } else {
14f8ab
             if (!gf_uuid_compare(replies[i].poststat.ia_gfid,
14f8ab
-                                 replies[source].poststat.ia_gfid))
14f8ab
+                                 replies[source].poststat.ia_gfid)) {
14f8ab
+                gf_msg_debug(this->name, 0, "skipping %s, no heal needed.",
14f8ab
+                             name);
14f8ab
                 continue;
14f8ab
+            }
14f8ab
 
14f8ab
             ret = afr_selfheal_recreate_entry(frame, i, source, sources,
14f8ab
                                               fd->inode, name, inode, replies);
14f8ab
@@ -839,7 +842,7 @@ afr_selfheal_entry_granular_dirent(xlator_t *subvol, gf_dirent_t *entry,
14f8ab
 
14f8ab
 out:
14f8ab
     loc_wipe(&loc;;
14f8ab
-    return 0;
14f8ab
+    return ret;
14f8ab
 }
14f8ab
 
14f8ab
 static int
14f8ab
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
14f8ab
index a72c494..bd17a82 100644
14f8ab
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
14f8ab
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
14f8ab
@@ -13181,6 +13181,19 @@ glusterd_enable_default_options(glusterd_volinfo_t *volinfo, char *option)
14f8ab
             goto out;
14f8ab
         }
14f8ab
     }
14f8ab
+
14f8ab
+    if ((conf->op_version >= GD_OP_VERSION_7_1) &&
14f8ab
+        (volinfo->status == GLUSTERD_STATUS_NONE)) {
14f8ab
+        ret = dict_set_dynstr_with_alloc(volinfo->dict,
14f8ab
+                                         "cluster.granular-entry-heal", "on");
14f8ab
+        if (ret) {
14f8ab
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
14f8ab
+                   "Failed to set option 'cluster.granular-entry-heal' "
14f8ab
+                   "on volume %s",
14f8ab
+                   volinfo->volname);
14f8ab
+            goto out;
14f8ab
+        }
14f8ab
+    }
14f8ab
 out:
14f8ab
     return ret;
14f8ab
 }
14f8ab
-- 
14f8ab
1.8.3.1
14f8ab