From 836b89d895539c2c214d145fcd74b25abd598530 Mon Sep 17 00:00:00 2001 From: Pranith Kumar K Date: Tue, 30 Jun 2015 23:01:36 +0530 Subject: [PATCH 200/200] cluster/ec: Make background healing optional behavior Backport of http://review.gluster.com/11473 Provide options to control number of active background heal count and qlen. Change-Id: Idc2419219d881f47e7d2e9bbc1dcdd999b372033 BUG: 1230612 Signed-off-by: Pranith Kumar K Reviewed-on: https://code.engineering.redhat.com/gerrit/52256 --- libglusterfs/src/globals.h | 12 +++-- tests/basic/afr/arbiter.t | 2 +- tests/basic/afr/client-side-heal.t | 10 ++-- tests/basic/afr/replace-brick-self-heal.t | 2 +- tests/basic/afr/root-squash-self-heal.t | 2 +- tests/basic/afr/self-heal.t | 16 +++--- tests/basic/afr/self-heald.t | 24 +++++----- tests/basic/afr/sparse-file-self-heal.t | 4 +- tests/basic/afr/split-brain-resolution.t | 4 +- tests/bugs/glusterfs/bug-861015-index.t | 2 +- tests/bugs/quota/afr-quota-xattr-mdata-heal.t | 2 +- tests/bugs/replicate/bug-1180545.t | 2 +- .../bug-1190069-afr-stale-index-entries.t | 2 +- tests/bugs/replicate/bug-918437-sh-mtime.t | 2 +- tests/volume.rc | 2 +- xlators/cluster/ec/src/ec-heal.c | 14 ++--- xlators/cluster/ec/src/ec.c | 50 ++++++++++++++++++- xlators/cluster/ec/src/ec.h | 2 + xlators/mgmt/glusterd/src/glusterd-volume-set.c | 8 +++ 19 files changed, 109 insertions(+), 53 deletions(-) diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h index 07185a8..9aca3c3 100644 --- a/libglusterfs/src/globals.h +++ b/libglusterfs/src/globals.h @@ -24,7 +24,7 @@ * RHS-2.1 u5 - 20105 * RHS-3.0 - 30000 * RHS-3.0.4 - 30004 - * RHGS-3.1 - 30702 + * RHGS-3.1 - 30703 * * * NOTE: @@ -42,9 +42,11 @@ */ #define GD_OP_VERSION_MIN 1 /* MIN is the fresh start op-version, mostly should not change */ -#define GD_OP_VERSION_MAX 30702 /* MAX VERSION is the maximum count in VME - table, should keep changing with - introduction of newer versions */ +#define GD_OP_VERSION_MAX GD_OP_VERSION_3_7_3 /* MAX VERSION is the maximum + count in VME table, should + keep changing with + introduction of newer + versions */ #define GD_OP_VERSION_RHS_3_0 30000 /* Op-Version of RHS 3.0 */ @@ -60,6 +62,8 @@ #define GD_OP_VERSION_3_7_2 30702 /* Op-version for GlusterFS 3.7.2 */ +#define GD_OP_VERSION_3_7_3 30703 /* Op-version for GlusterFS 3.7.3 */ + #include "xlator.h" /* THIS */ diff --git a/tests/basic/afr/arbiter.t b/tests/basic/afr/arbiter.t index 8a983fb..f06fdb1 100644 --- a/tests/basic/afr/arbiter.t +++ b/tests/basic/afr/arbiter.t @@ -60,7 +60,7 @@ EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 2 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 TEST $CLI volume heal $V0 -EXPECT 0 afr_get_pending_heal_count $V0 +EXPECT 0 get_pending_heal_count $V0 # I/O can resume again. TEST cat $M0/file diff --git a/tests/basic/afr/client-side-heal.t b/tests/basic/afr/client-side-heal.t index c9b3e35..18f7626 100644 --- a/tests/basic/afr/client-side-heal.t +++ b/tests/basic/afr/client-side-heal.t @@ -33,7 +33,7 @@ TEST chmod +x $M0/mdatafile #pending entry heal. Also causes pending metadata/data heals on file{1..5} TEST touch $M0/dir/file{1..5} -EXPECT 8 afr_get_pending_heal_count $V0 +EXPECT 8 get_pending_heal_count $V0 #After brick comes back up, access from client should not trigger heals TEST $CLI volume start $V0 force @@ -54,7 +54,7 @@ TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0; TEST ls $M0/dir #No heal must have happened -EXPECT 8 afr_get_pending_heal_count $V0 +EXPECT 8 get_pending_heal_count $V0 #Enable heal client side heal options and trigger heals TEST $CLI volume set $V0 cluster.data-self-heal on @@ -63,7 +63,7 @@ TEST $CLI volume set $V0 cluster.entry-self-heal on #Metadata heal is triggered by lookup without need for inode refresh. TEST ls $M0/mdatafile -EXPECT 7 afr_get_pending_heal_count $V0 +EXPECT 7 get_pending_heal_count $V0 #Inode refresh must trigger data and entry heals. #To trigger inode refresh for sure, the volume is unmounted and mounted each time. @@ -74,7 +74,7 @@ TEST cat $M0/datafile EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0; TEST ls $M0/dir -EXPECT 5 afr_get_pending_heal_count $V0 +EXPECT 5 get_pending_heal_count $V0 TEST cat $M0/dir/file1 TEST cat $M0/dir/file2 @@ -82,5 +82,5 @@ TEST cat $M0/dir/file3 TEST cat $M0/dir/file4 TEST cat $M0/dir/file5 -EXPECT 0 afr_get_pending_heal_count $V0 +EXPECT 0 get_pending_heal_count $V0 cleanup; diff --git a/tests/basic/afr/replace-brick-self-heal.t b/tests/basic/afr/replace-brick-self-heal.t index 8ced7df..fef671a 100644 --- a/tests/basic/afr/replace-brick-self-heal.t +++ b/tests/basic/afr/replace-brick-self-heal.t @@ -43,7 +43,7 @@ EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST $CLI volume heal $V0 # Wait for heal to complete -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 # Check if entry-heal has happened TEST diff <(ls $B0/${V0}0 | sort) <(ls $B0/${V0}1_new | sort) diff --git a/tests/basic/afr/root-squash-self-heal.t b/tests/basic/afr/root-squash-self-heal.t index fa9a163..8337432 100644 --- a/tests/basic/afr/root-squash-self-heal.t +++ b/tests/basic/afr/root-squash-self-heal.t @@ -20,6 +20,6 @@ echo abc > $M0/a TEST $CLI volume start $V0 force EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 find $M0 | xargs stat > /dev/null -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 cleanup diff --git a/tests/basic/afr/self-heal.t b/tests/basic/afr/self-heal.t index dbd8961..e1ac17c 100644 --- a/tests/basic/afr/self-heal.t +++ b/tests/basic/afr/self-heal.t @@ -53,7 +53,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST $CLI volume heal $V0 -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 #check all files created/deleted on brick1 are also replicated on brick 0 #(i.e. no reverse heal has happened) @@ -82,7 +82,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST $CLI volume heal $V0 -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 #check heal has happened in the correct direction TEST test -d $B0/brick0/file @@ -105,7 +105,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST $CLI volume heal $V0 -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 #check heal has happened in the correct direction EXPECT "777" stat -c %a $B0/brick0/file @@ -129,7 +129,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST $CLI volume heal $V0 -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 #check heal has happened in the correct direction EXPECT "$NEW_UID$NEW_GID" stat -c %u%g $B0/brick0/file @@ -160,7 +160,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST $CLI volume heal $V0 -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 #check heal has happened in the correct direction EXPECT 0 stat -c %s $B0/brick1/file @@ -183,7 +183,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST $CLI volume heal $V0 -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 #check heal has happened in the correct direction EXPECT "$GFID" gf_get_gfid_xattr $B0/brick0/file @@ -207,7 +207,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST $CLI volume heal $V0 -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 #check heal has happened in the correct direction TEST test -f $B0/brick0/hard_link_to_file @@ -233,7 +233,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST $CLI volume heal $V0 -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 TEST diff <(echo "user.myattr_1=\"My_attribute_1_modified\"") <(getfattr -n user.myattr_1 $B0/brick1/file|grep user.myattr_1) TEST diff <(echo "user.myattr_3=\"My_attribute_3\"") <(getfattr -n user.myattr_3 $B0/brick1/file|grep user.myattr_3) diff --git a/tests/basic/afr/self-heald.t b/tests/basic/afr/self-heald.t index ee0afaf..b8bee5c 100644 --- a/tests/basic/afr/self-heald.t +++ b/tests/basic/afr/self-heald.t @@ -68,7 +68,7 @@ done HEAL_FILES=$(($HEAL_FILES + 3)) cd ~ -EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0 +EXPECT "$HEAL_FILES" get_pending_heal_count $V0 #When bricks are down, it says Transport End point Not connected for them EXPECT "3" disconnected_brick_count $V0 @@ -78,12 +78,12 @@ EXPECT "3" disconnected_brick_count $V0 #replica pair. for i in {11..20}; do echo abc > $M0/$i; done HEAL_FILES=$(($HEAL_FILES + 10)) #count extra 10 files -EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0 +EXPECT "$HEAL_FILES" get_pending_heal_count $V0 #delete the files now, so that stale indices will remain. for i in {11..20}; do rm -f $M0/$i; done #After deleting files they should not appear in heal info HEAL_FILES=$(($HEAL_FILES - 10)) -EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0 +EXPECT "$HEAL_FILES" get_pending_heal_count $V0 TEST ! $CLI volume heal $V0 @@ -99,10 +99,10 @@ check_bricks_up $V0 TEST $CLI volume heal $V0 sleep 5 #Until the heal-statistics command implementation #check that this heals the contents partially -TEST [ $HEAL_FILES -gt $(afr_get_pending_heal_count $V0) ] +TEST [ $HEAL_FILES -gt $(get_pending_heal_count $V0) ] TEST $CLI volume heal $V0 full -EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN 30 "0" get_pending_heal_count $V0 #Test that ongoing IO is not considered as Pending heal (dd if=/dev/zero of=$M0/file1 bs=1k 2>/dev/null 1>/dev/null)& @@ -115,7 +115,7 @@ back_pid3=$!; back_pid4=$!; (dd if=/dev/zero of=$M0/file5 bs=1k 2>/dev/null 1>/dev/null)& back_pid5=$!; -EXPECT 0 afr_get_pending_heal_count $V0 +EXPECT 0 get_pending_heal_count $V0 kill -SIGTERM $back_pid1; kill -SIGTERM $back_pid2; kill -SIGTERM $back_pid3; @@ -132,13 +132,13 @@ TEST $CLI volume set $V0 cluster.data-self-heal off EXPECT "off" volume_option $V0 cluster.data-self-heal kill_multiple_bricks $V0 $H0 $B0 echo abc > $M0/f -EXPECT 1 afr_get_pending_heal_count $V0 +EXPECT 1 get_pending_heal_count $V0 TEST $CLI volume start $V0 force EXPECT_WITHIN 20 "Y" glustershd_up_status check_bricks_up $V0 TEST $CLI volume heal $V0 -EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN 30 "0" get_pending_heal_count $V0 TEST $CLI volume set $V0 cluster.data-self-heal on #METADATA @@ -147,13 +147,13 @@ EXPECT "off" volume_option $V0 cluster.metadata-self-heal kill_multiple_bricks $V0 $H0 $B0 TEST chmod 777 $M0/f -EXPECT 1 afr_get_pending_heal_count $V0 +EXPECT 1 get_pending_heal_count $V0 TEST $CLI volume start $V0 force EXPECT_WITHIN 20 "Y" glustershd_up_status check_bricks_up $V0 TEST $CLI volume heal $V0 -EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN 30 "0" get_pending_heal_count $V0 TEST $CLI volume set $V0 cluster.metadata-self-heal on #ENTRY @@ -163,13 +163,13 @@ kill_multiple_bricks $V0 $H0 $B0 TEST touch $M0/d/a # 4 if mtime/ctime is modified for d in bricks without a # 2 otherwise -PENDING=$( afr_get_pending_heal_count $V0 ) +PENDING=$( get_pending_heal_count $V0 ) TEST test $PENDING -eq 2 -o $PENDING -eq 4 TEST $CLI volume start $V0 force EXPECT_WITHIN 20 "Y" glustershd_up_status check_bricks_up $V0 TEST $CLI volume heal $V0 -EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN 30 "0" get_pending_heal_count $V0 TEST $CLI volume set $V0 cluster.entry-self-heal on #Negative test cases diff --git a/tests/basic/afr/sparse-file-self-heal.t b/tests/basic/afr/sparse-file-self-heal.t index 1bc915e..4101e6d 100644 --- a/tests/basic/afr/sparse-file-self-heal.t +++ b/tests/basic/afr/sparse-file-self-heal.t @@ -49,7 +49,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST gluster volume heal $V0 full -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 big_md5sum_0=$(md5sum $B0/${V0}0/big | awk '{print $1}') small_md5sum_0=$(md5sum $B0/${V0}0/small | awk '{print $1}') @@ -114,7 +114,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST gluster volume heal $V0 full -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 big_md5sum_0=$(md5sum $B0/${V0}0/big | awk '{print $1}') small_md5sum_0=$(md5sum $B0/${V0}0/small | awk '{print $1}') diff --git a/tests/basic/afr/split-brain-resolution.t b/tests/basic/afr/split-brain-resolution.t index fa1342e..84b2cc8 100644 --- a/tests/basic/afr/split-brain-resolution.t +++ b/tests/basic/afr/split-brain-resolution.t @@ -38,7 +38,7 @@ TEST $CLI volume start $V0 force EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 -EXPECT 4 afr_get_pending_heal_count $V0 +EXPECT 4 get_pending_heal_count $V0 TEST ! cat $M0/data-split-brain.txt TEST ! getfattr -n user.test $M0/metadata-split-brain.txt @@ -82,6 +82,6 @@ TEST setfattr -n replica.split-brain-heal-finalize -v $V0-client-1 $M0/data-spli EXPECT "brick0" get_text_xattr user.test $M0/metadata-split-brain.txt EXPECT "brick1_alive" cat $M0/data-split-brain.txt -EXPECT 0 afr_get_pending_heal_count $V0 +EXPECT 0 get_pending_heal_count $V0 cleanup; diff --git a/tests/bugs/glusterfs/bug-861015-index.t b/tests/bugs/glusterfs/bug-861015-index.t index 05f3e8b..74ffc45 100644 --- a/tests/bugs/glusterfs/bug-861015-index.t +++ b/tests/bugs/glusterfs/bug-861015-index.t @@ -25,7 +25,7 @@ done HEAL_FILES=$(($HEAL_FILES+3)) #count brick root distribute-subvol num of times cd ~ -EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0 +EXPECT "$HEAL_FILES" get_pending_heal_count $V0 TEST rm -f $M0/* EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 TEST $CLI volume heal $V0 info diff --git a/tests/bugs/quota/afr-quota-xattr-mdata-heal.t b/tests/bugs/quota/afr-quota-xattr-mdata-heal.t index 486b038..6aa2d83 100644 --- a/tests/bugs/quota/afr-quota-xattr-mdata-heal.t +++ b/tests/bugs/quota/afr-quota-xattr-mdata-heal.t @@ -55,7 +55,7 @@ TEST $CLI volume set $V0 cluster.self-heal-daemon on TEST $CLI volume start $V0 force EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 #Check external xattrs match EXPECT "bar" echo $(getfattr -d -m. -e text $B0/${V0}0/d | grep trusted.foo) diff --git a/tests/bugs/replicate/bug-1180545.t b/tests/bugs/replicate/bug-1180545.t index 748d5de..e953162 100644 --- a/tests/bugs/replicate/bug-1180545.t +++ b/tests/bugs/replicate/bug-1180545.t @@ -43,6 +43,6 @@ TEST $CLI volume heal $V0 EXPECT_WITHIN $HEAL_TIMEOUT '2' count_sh_entries $B0/brick0 EXPECT_WITHIN $HEAL_TIMEOUT '2' count_sh_entries $B0/brick1 #Two entries for DIR and two for FILE -EXPECT_WITHIN $HEAL_TIMEOUT "4" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "4" get_pending_heal_count $V0 TEST diff <(ls $B0/brick0/DIR) <(ls $B0/brick1/DIR) cleanup diff --git a/tests/bugs/replicate/bug-1190069-afr-stale-index-entries.t b/tests/bugs/replicate/bug-1190069-afr-stale-index-entries.t index a476563..2da960f 100644 --- a/tests/bugs/replicate/bug-1190069-afr-stale-index-entries.t +++ b/tests/bugs/replicate/bug-1190069-afr-stale-index-entries.t @@ -50,7 +50,7 @@ TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0 TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000000 $B0/${V0}1/mdatafile TEST $CLI volume heal $V0 -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 EXPECT_WITHIN $HEAL_TIMEOUT '0' echo $(count_sh_entries $B0/$V0"0") EXPECT_WITHIN $HEAL_TIMEOUT '0' echo $(count_sh_entries $B0/$V0"1") diff --git a/tests/bugs/replicate/bug-918437-sh-mtime.t b/tests/bugs/replicate/bug-918437-sh-mtime.t index 04ac02f..6a194b1 100644 --- a/tests/bugs/replicate/bug-918437-sh-mtime.t +++ b/tests/bugs/replicate/bug-918437-sh-mtime.t @@ -54,7 +54,7 @@ TEST stat $M0/b TEST gluster volume heal $V0 full EXPECT_WITHIN $HEAL_TIMEOUT "Y" file_exists $B0/gfs0/brick01/a EXPECT_WITHIN $HEAL_TIMEOUT "Y" file_exists $B0/gfs0/brick02/b -EXPECT_WITHIN $HEAL_TIMEOUT 0 afr_get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT 0 get_pending_heal_count $V0 size=`stat -c '%s' /etc/passwd` EXPECT $size stat -c '%s' $B0/gfs0/brick01/a diff --git a/tests/volume.rc b/tests/volume.rc index 47ca722..2d8dd72 100644 --- a/tests/volume.rc +++ b/tests/volume.rc @@ -216,7 +216,7 @@ function afr_get_changelog_xattr { echo $xval } -function afr_get_pending_heal_count { +function get_pending_heal_count { local vol=$1 gluster volume heal $vol info | grep "Number of entries" | awk '{ sum+=$4} END {print sum}' } diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c index 3168535..6ee1f9e 100644 --- a/xlators/cluster/ec/src/ec-heal.c +++ b/xlators/cluster/ec/src/ec-heal.c @@ -26,9 +26,6 @@ #include "syncop-utils.h" #include "cluster-syncop.h" -#define EC_MAX_BACKGROUND_HEALS 8 -#define EC_MAX_HEAL_WAITERS 128 - #define alloca0(size) ({void *__ptr; __ptr = alloca(size); memset(__ptr, 0, size); __ptr; }) #define EC_COUNT(array, max) ({int __i; int __res = 0; for (__i = 0; __i < max; __i++) if (array[__i]) __res++; __res; }) #define EC_INTERSECT(dst, src1, src2, max) ({int __i; for (__i = 0; __i < max; __i++) dst[__i] = src1[__i] && src2[__i]; }) @@ -2329,10 +2326,9 @@ __ec_dequeue_heals (ec_t *ec) if (list_empty (&ec->heal_waiting)) goto none; - if (ec->healers == EC_MAX_BACKGROUND_HEALS) + if ((ec->background_heals > 0) && (ec->healers >= ec->background_heals)) goto none; - GF_ASSERT (ec->healers < EC_MAX_BACKGROUND_HEALS); fop = list_entry(ec->heal_waiting.next, ec_fop_data_t, healer); ec->heal_waiters--; list_del_init(&fop->healer); @@ -2400,12 +2396,14 @@ ec_heal_throttle (xlator_t *this, ec_fop_data_t *fop) LOCK (&ec->lock); { - if (ec->heal_waiters >= EC_MAX_HEAL_WAITERS) { - can_heal = _gf_false; - } else { + if ((ec->background_heals > 0) && + (ec->heal_wait_qlen + ec->background_heals) > + (ec->heal_waiters + ec->healers)) { list_add_tail(&fop->healer, &ec->heal_waiting); ec->heal_waiters++; fop = __ec_dequeue_heals (ec); + } else { + can_heal = _gf_false; } } UNLOCK (&ec->lock); diff --git a/xlators/cluster/ec/src/ec.c b/xlators/cluster/ec/src/ec.c index dd51630..e28f402 100644 --- a/xlators/cluster/ec/src/ec.c +++ b/xlators/cluster/ec/src/ec.c @@ -219,15 +219,35 @@ int32_t mem_acct_init(xlator_t * this) return 0; } +void +ec_configure_background_heal_opts (ec_t *ec, int background_heals, + int heal_wait_qlen) +{ + if (background_heals == 0) { + ec->heal_wait_qlen = 0; + } else { + ec->heal_wait_qlen = heal_wait_qlen; + } + ec->background_heals = background_heals; +} + int32_t reconfigure (xlator_t *this, dict_t *options) { - ec_t *ec = this->private; + ec_t *ec = this->private; + uint32_t heal_wait_qlen = 0; + uint32_t background_heals = 0; - GF_OPTION_RECONF ("self-heal-daemon", ec->shd.enabled, options, bool, failed); + GF_OPTION_RECONF ("self-heal-daemon", ec->shd.enabled, options, bool, + failed); GF_OPTION_RECONF ("iam-self-heal-daemon", ec->shd.iamshd, options, bool, failed); - + GF_OPTION_RECONF ("background-heals", background_heals, options, + uint32, failed); + GF_OPTION_RECONF ("heal-wait-qlength", heal_wait_qlen, options, + uint32, failed); + ec_configure_background_heal_opts (ec, background_heals, + heal_wait_qlen); return 0; failed: return -1; @@ -577,6 +597,10 @@ init (xlator_t *this) ec_method_initialize(); GF_OPTION_INIT ("self-heal-daemon", ec->shd.enabled, bool, failed); GF_OPTION_INIT ("iam-self-heal-daemon", ec->shd.iamshd, bool, failed); + GF_OPTION_INIT ("background-heals", ec->background_heals, uint32, failed); + GF_OPTION_INIT ("heal-wait-qlength", ec->heal_wait_qlen, uint32, failed); + ec_configure_background_heal_opts (ec, ec->background_heals, + ec->heal_wait_qlen); if (ec->shd.iamshd) ec_selfheal_daemon_init (this); @@ -1188,6 +1212,10 @@ int32_t ec_dump_private(xlator_t *this) gf_proc_dump_write("childs_up", "%u", ec->xl_up_count); gf_proc_dump_write("childs_up_mask", "%s", ec_bin(tmp, sizeof(tmp), ec->xl_up, ec->nodes)); + gf_proc_dump_write("background-heals", "%d", ec->background_heals); + gf_proc_dump_write("heal-wait-qlength", "%d", ec->heal_wait_qlen); + gf_proc_dump_write("healers", "%d", ec->healers); + gf_proc_dump_write("heal-waiters", "%d", ec->heal_waiters); return 0; } @@ -1271,5 +1299,21 @@ struct volume_options options[] = "translator is running as part of self-heal-daemon " "or not." }, + { .key = {"background-heals"}, + .type = GF_OPTION_TYPE_INT, + .min = 0,/*Disabling background heals*/ + .max = 256, + .default_value = "8", + .description = "This option can be used to control number of parallel" + " heals", + }, + { .key = {"heal-wait-qlength"}, + .type = GF_OPTION_TYPE_INT, + .min = 0, + .max = 65536, /*Around 100MB as of now with sizeof(ec_fop_data_t) at 1800*/ + .default_value = "128", + .description = "This option can be used to control number of heals" + " that can wait", + }, { } }; diff --git a/xlators/cluster/ec/src/ec.h b/xlators/cluster/ec/src/ec.h index 7f14020..f335fd5 100644 --- a/xlators/cluster/ec/src/ec.h +++ b/xlators/cluster/ec/src/ec.h @@ -47,6 +47,8 @@ struct _ec gf_lock_t lock; gf_timer_t * timer; gf_boolean_t shutdown; + uint32_t background_heals; + uint32_t heal_wait_qlen; struct list_head pending_fops; struct list_head heal_waiting; struct list_head healing; diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c index 7bbd729..b536d18 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c @@ -2015,6 +2015,14 @@ struct volopt_map_entry glusterd_volopt_map[] = { .voltype = "features/upcall", .op_version = GD_OP_VERSION_3_7_0, }, + { .key = "disperse.background-heals", + .voltype = "cluster/disperse", + .op_version = GD_OP_VERSION_3_7_3, + }, + { .key = "disperse.heal-wait-qlength", + .voltype = "cluster/disperse", + .op_version = GD_OP_VERSION_3_7_3, + }, { .key = NULL } }; -- 1.7.1