From 836b89d895539c2c214d145fcd74b25abd598530 Mon Sep 17 00:00:00 2001

From: Pranith Kumar K <pkarampu@redhat.com>

Date: Tue, 30 Jun 2015 23:01:36 +0530

Subject: [PATCH 200/200] cluster/ec: Make background healing optional behavior

        Backport of http://review.gluster.com/11473

Provide options to control number of active background heal count and qlen.

Change-Id: Idc2419219d881f47e7d2e9bbc1dcdd999b372033

BUG: 1230612

Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/52256

---

 libglusterfs/src/globals.h                         |   12 +++--

 tests/basic/afr/arbiter.t                          |    2 +-

 tests/basic/afr/client-side-heal.t                 |   10 ++--

 tests/basic/afr/replace-brick-self-heal.t          |    2 +-

 tests/basic/afr/root-squash-self-heal.t            |    2 +-

 tests/basic/afr/self-heal.t                        |   16 +++---

 tests/basic/afr/self-heald.t                       |   24 +++++-----

 tests/basic/afr/sparse-file-self-heal.t            |    4 +-

 tests/basic/afr/split-brain-resolution.t           |    4 +-

 tests/bugs/glusterfs/bug-861015-index.t            |    2 +-

 tests/bugs/quota/afr-quota-xattr-mdata-heal.t      |    2 +-

 tests/bugs/replicate/bug-1180545.t                 |    2 +-

 .../bug-1190069-afr-stale-index-entries.t          |    2 +-

 tests/bugs/replicate/bug-918437-sh-mtime.t         |    2 +-

 tests/volume.rc                                    |    2 +-

 xlators/cluster/ec/src/ec-heal.c                   |   14 ++---

 xlators/cluster/ec/src/ec.c                        |   50 ++++++++++++++++++-

 xlators/cluster/ec/src/ec.h                        |    2 +

 xlators/mgmt/glusterd/src/glusterd-volume-set.c    |    8 +++

 19 files changed, 109 insertions(+), 53 deletions(-)

diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h

index 07185a8..9aca3c3 100644

--- a/libglusterfs/src/globals.h

+++ b/libglusterfs/src/globals.h

@@ -24,7 +24,7 @@

  * RHS-2.1 u5   - 20105

  * RHS-3.0      - 30000

  * RHS-3.0.4    - 30004

- * RHGS-3.1     - 30702

+ * RHGS-3.1     - 30703

  *

  *

  * NOTE:

@@ -42,9 +42,11 @@

  */

 #define GD_OP_VERSION_MIN  1 /* MIN is the fresh start op-version, mostly

                                 should not change */

-#define GD_OP_VERSION_MAX  30702 /* MAX VERSION is the maximum count in VME

-                                    table, should keep changing with

-                                    introduction of newer versions */

+#define GD_OP_VERSION_MAX  GD_OP_VERSION_3_7_3 /* MAX VERSION is the maximum

+                                                  count in VME table, should

+                                                  keep changing with

+                                                  introduction of newer

+                                                  versions */

 #define GD_OP_VERSION_RHS_3_0    30000 /* Op-Version of RHS 3.0 */

@@ -60,6 +62,8 @@

 #define GD_OP_VERSION_3_7_2    30702 /* Op-version for GlusterFS 3.7.2 */

+#define GD_OP_VERSION_3_7_3    30703 /* Op-version for GlusterFS 3.7.3 */

+

 #include "xlator.h"

 /* THIS */

diff --git a/tests/basic/afr/arbiter.t b/tests/basic/afr/arbiter.t

index 8a983fb..f06fdb1 100644

--- a/tests/basic/afr/arbiter.t

+++ b/tests/basic/afr/arbiter.t

@@ -60,7 +60,7 @@ EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 2

 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1

 TEST $CLI volume heal $V0

-EXPECT 0 afr_get_pending_heal_count $V0

+EXPECT 0 get_pending_heal_count $V0

 # I/O can resume again.

 TEST cat $M0/file

diff --git a/tests/basic/afr/client-side-heal.t b/tests/basic/afr/client-side-heal.t

index c9b3e35..18f7626 100644

--- a/tests/basic/afr/client-side-heal.t

+++ b/tests/basic/afr/client-side-heal.t

@@ -33,7 +33,7 @@ TEST chmod +x $M0/mdatafile

 #pending entry heal. Also causes pending metadata/data heals on file{1..5}

 TEST touch $M0/dir/file{1..5}

-EXPECT 8 afr_get_pending_heal_count $V0

+EXPECT 8 get_pending_heal_count $V0

 #After brick comes back up, access from client should not trigger heals

 TEST $CLI volume start $V0 force

@@ -54,7 +54,7 @@ TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0;

 TEST ls $M0/dir

 #No heal must have happened

-EXPECT 8 afr_get_pending_heal_count $V0

+EXPECT 8 get_pending_heal_count $V0

 #Enable heal client side heal options and trigger heals

 TEST $CLI volume set $V0 cluster.data-self-heal on

@@ -63,7 +63,7 @@ TEST $CLI volume set $V0 cluster.entry-self-heal on

 #Metadata heal is triggered by lookup without need for inode refresh.

 TEST ls $M0/mdatafile

-EXPECT 7 afr_get_pending_heal_count $V0

+EXPECT 7 get_pending_heal_count $V0

 #Inode refresh must trigger data and entry heals.

 #To trigger inode refresh for sure, the volume is unmounted and mounted each time.

@@ -74,7 +74,7 @@ TEST cat $M0/datafile

 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0

 TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0;

 TEST ls $M0/dir

-EXPECT 5 afr_get_pending_heal_count $V0

+EXPECT 5 get_pending_heal_count $V0

 TEST cat  $M0/dir/file1

 TEST cat  $M0/dir/file2

@@ -82,5 +82,5 @@ TEST cat  $M0/dir/file3

 TEST cat  $M0/dir/file4

 TEST cat  $M0/dir/file5

-EXPECT 0 afr_get_pending_heal_count $V0

+EXPECT 0 get_pending_heal_count $V0

 cleanup;

diff --git a/tests/basic/afr/replace-brick-self-heal.t b/tests/basic/afr/replace-brick-self-heal.t

index 8ced7df..fef671a 100644

--- a/tests/basic/afr/replace-brick-self-heal.t

+++ b/tests/basic/afr/replace-brick-self-heal.t

@@ -43,7 +43,7 @@ EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 TEST $CLI volume heal $V0

 # Wait for heal to complete

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 # Check if entry-heal has happened

 TEST diff <(ls $B0/${V0}0 | sort) <(ls $B0/${V0}1_new | sort)

diff --git a/tests/basic/afr/root-squash-self-heal.t b/tests/basic/afr/root-squash-self-heal.t

index fa9a163..8337432 100644

--- a/tests/basic/afr/root-squash-self-heal.t

+++ b/tests/basic/afr/root-squash-self-heal.t

@@ -20,6 +20,6 @@ echo abc > $M0/a

 TEST $CLI volume start $V0 force

 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0

 find $M0 | xargs stat > /dev/null

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 cleanup

diff --git a/tests/basic/afr/self-heal.t b/tests/basic/afr/self-heal.t

index dbd8961..e1ac17c 100644

--- a/tests/basic/afr/self-heal.t

+++ b/tests/basic/afr/self-heal.t

@@ -53,7 +53,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 TEST $CLI volume heal $V0

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 #check all files created/deleted on brick1 are also replicated on brick 0

 #(i.e. no reverse heal has happened)

@@ -82,7 +82,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 TEST $CLI volume heal $V0

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 #check heal has happened in the correct direction

 TEST test -d $B0/brick0/file

@@ -105,7 +105,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 TEST $CLI volume heal $V0

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 #check heal has happened in the correct direction

 EXPECT "777" stat -c %a $B0/brick0/file

@@ -129,7 +129,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 TEST $CLI volume heal $V0

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 #check heal has happened in the correct direction

 EXPECT "$NEW_UID$NEW_GID" stat -c %u%g $B0/brick0/file

@@ -160,7 +160,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 TEST $CLI volume heal $V0

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 #check heal has happened in the correct direction

 EXPECT 0 stat -c %s $B0/brick1/file

@@ -183,7 +183,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 TEST $CLI volume heal $V0

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 #check heal has happened in the correct direction

 EXPECT "$GFID" gf_get_gfid_xattr $B0/brick0/file

@@ -207,7 +207,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 TEST $CLI volume heal $V0

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 #check heal has happened in the correct direction

 TEST test -f $B0/brick0/hard_link_to_file

@@ -233,7 +233,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 TEST $CLI volume heal $V0

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 TEST diff <(echo "user.myattr_1=\"My_attribute_1_modified\"") <(getfattr -n user.myattr_1 $B0/brick1/file|grep user.myattr_1)

 TEST diff <(echo "user.myattr_3=\"My_attribute_3\"") <(getfattr -n user.myattr_3 $B0/brick1/file|grep user.myattr_3)

diff --git a/tests/basic/afr/self-heald.t b/tests/basic/afr/self-heald.t

index ee0afaf..b8bee5c 100644

--- a/tests/basic/afr/self-heald.t

+++ b/tests/basic/afr/self-heald.t

@@ -68,7 +68,7 @@ done

 HEAL_FILES=$(($HEAL_FILES + 3))

 cd ~

-EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0

+EXPECT "$HEAL_FILES" get_pending_heal_count $V0

 #When bricks are down, it says Transport End point Not connected for them

 EXPECT "3" disconnected_brick_count $V0

@@ -78,12 +78,12 @@ EXPECT "3" disconnected_brick_count $V0

 #replica pair.

 for i in {11..20}; do echo abc > $M0/$i; done

 HEAL_FILES=$(($HEAL_FILES + 10)) #count extra 10 files

-EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0

+EXPECT "$HEAL_FILES" get_pending_heal_count $V0

 #delete the files now, so that stale indices will remain.

 for i in {11..20}; do rm -f $M0/$i; done

 #After deleting files they should not appear in heal info

 HEAL_FILES=$(($HEAL_FILES - 10))

-EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0

+EXPECT "$HEAL_FILES" get_pending_heal_count $V0

 TEST ! $CLI volume heal $V0

@@ -99,10 +99,10 @@ check_bricks_up $V0

 TEST $CLI volume heal $V0

 sleep 5 #Until the heal-statistics command implementation

 #check that this heals the contents partially

-TEST [ $HEAL_FILES -gt $(afr_get_pending_heal_count $V0) ]

+TEST [ $HEAL_FILES -gt $(get_pending_heal_count $V0) ]

 TEST $CLI volume heal $V0 full

-EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN 30 "0" get_pending_heal_count $V0

 #Test that ongoing IO is not considered as Pending heal

 (dd if=/dev/zero of=$M0/file1 bs=1k 2>/dev/null 1>/dev/null)&

@@ -115,7 +115,7 @@ back_pid3=$!;

 back_pid4=$!;

 (dd if=/dev/zero of=$M0/file5 bs=1k 2>/dev/null 1>/dev/null)&

 back_pid5=$!;

-EXPECT 0 afr_get_pending_heal_count $V0

+EXPECT 0 get_pending_heal_count $V0

 kill -SIGTERM $back_pid1;

 kill -SIGTERM $back_pid2;

 kill -SIGTERM $back_pid3;

@@ -132,13 +132,13 @@ TEST $CLI volume set $V0 cluster.data-self-heal off

 EXPECT "off" volume_option $V0 cluster.data-self-heal

 kill_multiple_bricks $V0 $H0 $B0

 echo abc > $M0/f

-EXPECT 1 afr_get_pending_heal_count $V0

+EXPECT 1 get_pending_heal_count $V0

 TEST $CLI volume start $V0 force

 EXPECT_WITHIN 20 "Y" glustershd_up_status

 check_bricks_up $V0

 TEST $CLI volume heal $V0

-EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN 30 "0" get_pending_heal_count $V0

 TEST $CLI volume set $V0 cluster.data-self-heal on

 #METADATA

@@ -147,13 +147,13 @@ EXPECT "off" volume_option $V0 cluster.metadata-self-heal

 kill_multiple_bricks $V0 $H0 $B0

 TEST chmod 777 $M0/f

-EXPECT 1 afr_get_pending_heal_count $V0

+EXPECT 1 get_pending_heal_count $V0

 TEST $CLI volume start $V0 force

 EXPECT_WITHIN 20 "Y" glustershd_up_status

 check_bricks_up $V0

 TEST $CLI volume heal $V0

-EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN 30 "0" get_pending_heal_count $V0

 TEST $CLI volume set $V0 cluster.metadata-self-heal on

 #ENTRY

@@ -163,13 +163,13 @@ kill_multiple_bricks $V0 $H0 $B0

 TEST touch $M0/d/a

 # 4 if mtime/ctime is modified for d in bricks without a

 # 2 otherwise

-PENDING=$( afr_get_pending_heal_count $V0 )

+PENDING=$( get_pending_heal_count $V0 )

 TEST test $PENDING -eq 2 -o $PENDING -eq 4

 TEST $CLI volume start $V0 force

 EXPECT_WITHIN 20 "Y" glustershd_up_status

 check_bricks_up $V0

 TEST $CLI volume heal $V0

-EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN 30 "0" get_pending_heal_count $V0

 TEST $CLI volume set $V0 cluster.entry-self-heal on

 #Negative test cases

diff --git a/tests/basic/afr/sparse-file-self-heal.t b/tests/basic/afr/sparse-file-self-heal.t

index 1bc915e..4101e6d 100644

--- a/tests/basic/afr/sparse-file-self-heal.t

+++ b/tests/basic/afr/sparse-file-self-heal.t

@@ -49,7 +49,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 TEST gluster volume heal $V0 full

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 big_md5sum_0=$(md5sum $B0/${V0}0/big | awk '{print $1}')

 small_md5sum_0=$(md5sum $B0/${V0}0/small | awk '{print $1}')

@@ -114,7 +114,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1

 TEST gluster volume heal $V0 full

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 big_md5sum_0=$(md5sum $B0/${V0}0/big | awk '{print $1}')

 small_md5sum_0=$(md5sum $B0/${V0}0/small | awk '{print $1}')

diff --git a/tests/basic/afr/split-brain-resolution.t b/tests/basic/afr/split-brain-resolution.t

index fa1342e..84b2cc8 100644

--- a/tests/basic/afr/split-brain-resolution.t

+++ b/tests/basic/afr/split-brain-resolution.t

@@ -38,7 +38,7 @@ TEST $CLI volume start $V0 force

 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0

 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1

-EXPECT 4 afr_get_pending_heal_count $V0

+EXPECT 4 get_pending_heal_count $V0

 TEST ! cat $M0/data-split-brain.txt

 TEST ! getfattr -n user.test $M0/metadata-split-brain.txt

@@ -82,6 +82,6 @@ TEST setfattr -n replica.split-brain-heal-finalize -v $V0-client-1 $M0/data-spli

 EXPECT "brick0" get_text_xattr user.test $M0/metadata-split-brain.txt

 EXPECT "brick1_alive" cat $M0/data-split-brain.txt

-EXPECT 0 afr_get_pending_heal_count $V0

+EXPECT 0 get_pending_heal_count $V0

 cleanup;

diff --git a/tests/bugs/glusterfs/bug-861015-index.t b/tests/bugs/glusterfs/bug-861015-index.t

index 05f3e8b..74ffc45 100644

--- a/tests/bugs/glusterfs/bug-861015-index.t

+++ b/tests/bugs/glusterfs/bug-861015-index.t

@@ -25,7 +25,7 @@ done

 HEAL_FILES=$(($HEAL_FILES+3)) #count brick root distribute-subvol num of times

 cd ~

-EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0

+EXPECT "$HEAL_FILES" get_pending_heal_count $V0

 TEST rm -f $M0/*

 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0

 TEST $CLI volume heal $V0 info

diff --git a/tests/bugs/quota/afr-quota-xattr-mdata-heal.t b/tests/bugs/quota/afr-quota-xattr-mdata-heal.t

index 486b038..6aa2d83 100644

--- a/tests/bugs/quota/afr-quota-xattr-mdata-heal.t

+++ b/tests/bugs/quota/afr-quota-xattr-mdata-heal.t

@@ -55,7 +55,7 @@ TEST $CLI volume set $V0 cluster.self-heal-daemon on

 TEST $CLI volume start $V0 force

 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status

 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 #Check external xattrs match

 EXPECT "bar" echo $(getfattr -d -m. -e text $B0/${V0}0/d | grep trusted.foo)

diff --git a/tests/bugs/replicate/bug-1180545.t b/tests/bugs/replicate/bug-1180545.t

index 748d5de..e953162 100644

--- a/tests/bugs/replicate/bug-1180545.t

+++ b/tests/bugs/replicate/bug-1180545.t

@@ -43,6 +43,6 @@ TEST $CLI volume heal $V0

 EXPECT_WITHIN $HEAL_TIMEOUT '2' count_sh_entries $B0/brick0

 EXPECT_WITHIN $HEAL_TIMEOUT '2' count_sh_entries $B0/brick1

 #Two entries for DIR and two for FILE

-EXPECT_WITHIN $HEAL_TIMEOUT "4" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "4" get_pending_heal_count $V0

 TEST diff <(ls $B0/brick0/DIR) <(ls $B0/brick1/DIR)

 cleanup

diff --git a/tests/bugs/replicate/bug-1190069-afr-stale-index-entries.t b/tests/bugs/replicate/bug-1190069-afr-stale-index-entries.t

index a476563..2da960f 100644

--- a/tests/bugs/replicate/bug-1190069-afr-stale-index-entries.t

+++ b/tests/bugs/replicate/bug-1190069-afr-stale-index-entries.t

@@ -50,7 +50,7 @@ TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0

 TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000000 $B0/${V0}1/mdatafile

 TEST $CLI volume heal $V0

-EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0

 EXPECT_WITHIN $HEAL_TIMEOUT '0' echo $(count_sh_entries $B0/$V0"0")

 EXPECT_WITHIN $HEAL_TIMEOUT '0' echo $(count_sh_entries $B0/$V0"1")

diff --git a/tests/bugs/replicate/bug-918437-sh-mtime.t b/tests/bugs/replicate/bug-918437-sh-mtime.t

index 04ac02f..6a194b1 100644

--- a/tests/bugs/replicate/bug-918437-sh-mtime.t

+++ b/tests/bugs/replicate/bug-918437-sh-mtime.t

@@ -54,7 +54,7 @@ TEST stat $M0/b

 TEST gluster volume heal $V0 full

 EXPECT_WITHIN $HEAL_TIMEOUT "Y" file_exists $B0/gfs0/brick01/a

 EXPECT_WITHIN $HEAL_TIMEOUT "Y" file_exists $B0/gfs0/brick02/b

-EXPECT_WITHIN $HEAL_TIMEOUT 0 afr_get_pending_heal_count $V0

+EXPECT_WITHIN $HEAL_TIMEOUT 0 get_pending_heal_count $V0

 size=`stat -c '%s' /etc/passwd`

 EXPECT $size stat -c '%s' $B0/gfs0/brick01/a

diff --git a/tests/volume.rc b/tests/volume.rc

index 47ca722..2d8dd72 100644

--- a/tests/volume.rc

+++ b/tests/volume.rc

@@ -216,7 +216,7 @@ function afr_get_changelog_xattr {

         echo $xval

 }

-function afr_get_pending_heal_count {

+function get_pending_heal_count {

         local vol=$1

         gluster volume heal $vol info | grep "Number of entries" | awk '{ sum+=$4} END {print sum}'

 }

diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c

index 3168535..6ee1f9e 100644

--- a/xlators/cluster/ec/src/ec-heal.c

+++ b/xlators/cluster/ec/src/ec-heal.c

@@ -26,9 +26,6 @@

 #include "syncop-utils.h"

 #include "cluster-syncop.h"

-#define EC_MAX_BACKGROUND_HEALS 8

-#define EC_MAX_HEAL_WAITERS 128

-

 #define alloca0(size) ({void *__ptr; __ptr = alloca(size); memset(__ptr, 0, size); __ptr; })

 #define EC_COUNT(array, max) ({int __i; int __res = 0; for (__i = 0; __i < max; __i++) if (array[__i]) __res++; __res; })

 #define EC_INTERSECT(dst, src1, src2, max) ({int __i; for (__i = 0; __i < max; __i++) dst[__i] = src1[__i] && src2[__i]; })

@@ -2329,10 +2326,9 @@ __ec_dequeue_heals (ec_t *ec)

         if (list_empty (&ec->heal_waiting))

                 goto none;

-        if (ec->healers == EC_MAX_BACKGROUND_HEALS)

+        if ((ec->background_heals > 0) && (ec->healers >= ec->background_heals))

                 goto none;

-        GF_ASSERT (ec->healers < EC_MAX_BACKGROUND_HEALS);

         fop = list_entry(ec->heal_waiting.next, ec_fop_data_t, healer);

         ec->heal_waiters--;

         list_del_init(&fop->healer);

@@ -2400,12 +2396,14 @@ ec_heal_throttle (xlator_t *this, ec_fop_data_t *fop)

                 LOCK (&ec->lock);

                 {

-                        if (ec->heal_waiters >= EC_MAX_HEAL_WAITERS) {

-                                can_heal = _gf_false;

-                        } else {

+                        if ((ec->background_heals > 0) &&

+                            (ec->heal_wait_qlen + ec->background_heals) >

+                                             (ec->heal_waiters + ec->healers)) {

                                 list_add_tail(&fop->healer, &ec->heal_waiting);

                                 ec->heal_waiters++;

                                 fop = __ec_dequeue_heals (ec);

+                        } else {

+                                can_heal = _gf_false;

                         }

                 }

                 UNLOCK (&ec->lock);

diff --git a/xlators/cluster/ec/src/ec.c b/xlators/cluster/ec/src/ec.c

index dd51630..e28f402 100644

--- a/xlators/cluster/ec/src/ec.c

+++ b/xlators/cluster/ec/src/ec.c

@@ -219,15 +219,35 @@ int32_t mem_acct_init(xlator_t * this)

     return 0;

 }

+void

+ec_configure_background_heal_opts (ec_t *ec, int background_heals,

+                                   int heal_wait_qlen)

+{

+        if (background_heals == 0) {

+                ec->heal_wait_qlen = 0;

+        } else {

+                ec->heal_wait_qlen = heal_wait_qlen;

+        }

+        ec->background_heals = background_heals;

+}

+

 int32_t

 reconfigure (xlator_t *this, dict_t *options)

 {

-        ec_t *ec = this->private;

+        ec_t     *ec              = this->private;

+        uint32_t heal_wait_qlen   = 0;

+        uint32_t background_heals = 0;

-        GF_OPTION_RECONF ("self-heal-daemon", ec->shd.enabled, options, bool, failed);

+        GF_OPTION_RECONF ("self-heal-daemon", ec->shd.enabled, options, bool,

+                          failed);

         GF_OPTION_RECONF ("iam-self-heal-daemon", ec->shd.iamshd, options,

                           bool, failed);

-

+        GF_OPTION_RECONF ("background-heals", background_heals, options,

+                          uint32, failed);

+        GF_OPTION_RECONF ("heal-wait-qlength", heal_wait_qlen, options,

+                          uint32, failed);

+        ec_configure_background_heal_opts (ec, background_heals,

+                                           heal_wait_qlen);

         return 0;

 failed:

         return -1;

@@ -577,6 +597,10 @@ init (xlator_t *this)

     ec_method_initialize();

     GF_OPTION_INIT ("self-heal-daemon", ec->shd.enabled, bool, failed);

     GF_OPTION_INIT ("iam-self-heal-daemon", ec->shd.iamshd, bool, failed);

+    GF_OPTION_INIT ("background-heals", ec->background_heals, uint32, failed);

+    GF_OPTION_INIT ("heal-wait-qlength", ec->heal_wait_qlen, uint32, failed);

+    ec_configure_background_heal_opts (ec, ec->background_heals,

+                                       ec->heal_wait_qlen);

     if (ec->shd.iamshd)

             ec_selfheal_daemon_init (this);

@@ -1188,6 +1212,10 @@ int32_t ec_dump_private(xlator_t *this)

     gf_proc_dump_write("childs_up", "%u", ec->xl_up_count);

     gf_proc_dump_write("childs_up_mask", "%s",

                        ec_bin(tmp, sizeof(tmp), ec->xl_up, ec->nodes));

+    gf_proc_dump_write("background-heals", "%d", ec->background_heals);

+    gf_proc_dump_write("heal-wait-qlength", "%d", ec->heal_wait_qlen);

+    gf_proc_dump_write("healers", "%d", ec->healers);

+    gf_proc_dump_write("heal-waiters", "%d", ec->heal_waiters);

     return 0;

 }

@@ -1271,5 +1299,21 @@ struct volume_options options[] =

                      "translator is running as part of self-heal-daemon "

                      "or not."

     },

+    { .key = {"background-heals"},

+      .type = GF_OPTION_TYPE_INT,

+      .min = 0,/*Disabling background heals*/

+      .max = 256,

+      .default_value = "8",

+      .description = "This option can be used to control number of parallel"

+                     " heals",

+    },

+    { .key = {"heal-wait-qlength"},

+      .type = GF_OPTION_TYPE_INT,

+      .min = 0,

+      .max = 65536, /*Around 100MB as of now with sizeof(ec_fop_data_t) at 1800*/

+      .default_value = "128",

+      .description = "This option can be used to control number of heals"

+                     " that can wait",

+    },

     { }

 };

diff --git a/xlators/cluster/ec/src/ec.h b/xlators/cluster/ec/src/ec.h

index 7f14020..f335fd5 100644

--- a/xlators/cluster/ec/src/ec.h

+++ b/xlators/cluster/ec/src/ec.h

@@ -47,6 +47,8 @@ struct _ec

     gf_lock_t         lock;

     gf_timer_t *      timer;

     gf_boolean_t      shutdown;

+    uint32_t          background_heals;

+    uint32_t          heal_wait_qlen;

     struct list_head  pending_fops;

     struct list_head  heal_waiting;

     struct list_head  healing;

diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c

index 7bbd729..b536d18 100644

--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c

+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c

@@ -2015,6 +2015,14 @@ struct volopt_map_entry glusterd_volopt_map[] = {

           .voltype     = "features/upcall",

           .op_version  = GD_OP_VERSION_3_7_0,

         },

+        { .key         = "disperse.background-heals",

+          .voltype     = "cluster/disperse",

+          .op_version  = GD_OP_VERSION_3_7_3,

+        },

+        { .key         = "disperse.heal-wait-qlength",

+          .voltype     = "cluster/disperse",

+          .op_version  = GD_OP_VERSION_3_7_3,

+        },

         { .key         = NULL

         }

 };

-- 

1.7.1