From a468ff0af57528dbd7a711f985c0524251ec90d3 Mon Sep 17 00:00:00 2001

From: Ravishankar N <ravishankar@redhat.com>

Date: Tue, 5 Apr 2016 11:40:05 +0530

Subject: [PATCH 40/80] afr: Add throttled background client-side heals

Backport of: http://review.gluster.org/13207

If a heal is needed after inode refresh (lookup, read_txn), launch it in

the background instead of blocking the fop (that triggered refresh)

until the heal happens.

afr_replies_interpret() is modified such that the heal is

launched only if atleast one sink brick is up.

Max. no of heals that can happen in parallel is configurable via the

'background-self-heal-count' volume option. Any number greater than that

is put in a wait queue whose length is configurable via

'heal-wait-queue-leng' volume option. If the wait queue is also full,

further heals will be ignored.

Default values:  background-self-heal-count=8, heal-wait-queue-leng=128

Change-Id: Ief20b915f8b3064dfbde41e9216b080de45f31f5

BUG: 1300875

Signed-off-by: Ravishankar N <ravishankar@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/71393

Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

Tested-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

---

 tests/basic/afr/client-side-heal.t              |    1 +

 tests/bugs/glusterd/859927/repl.t               |    4 +-

 tests/bugs/quota/bug-1035576.t                  |    1 -

 tests/bugs/replicate/bug-802417.t               |    5 +-

 tests/bugs/replicate/bug-977797.t               |   52 +++++------

 xlators/cluster/afr/src/afr-common.c            |   89 +++++++++----------

 xlators/cluster/afr/src/afr-dir-write.c         |    2 +-

 xlators/cluster/afr/src/afr-self-heal-common.c  |  110 +++++++++++++++++++++++

 xlators/cluster/afr/src/afr-self-heal.h         |    3 +

 xlators/cluster/afr/src/afr.c                   |   26 +++++-

 xlators/cluster/afr/src/afr.h                   |   28 +++++--

 xlators/mgmt/glusterd/src/glusterd-volume-set.c |    8 ++

 12 files changed, 237 insertions(+), 92 deletions(-)

diff --git a/tests/basic/afr/client-side-heal.t b/tests/basic/afr/client-side-heal.t

index 18f7626..d87f4b1 100644

--- a/tests/basic/afr/client-side-heal.t

+++ b/tests/basic/afr/client-side-heal.t

@@ -70,6 +70,7 @@ EXPECT 7 get_pending_heal_count $V0

 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0

 TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0;

 TEST cat $M0/datafile

+EXPECT_WITHIN $HEAL_TIMEOUT 6 get_pending_heal_count $V0

 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0

 TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0;

diff --git a/tests/bugs/glusterd/859927/repl.t b/tests/bugs/glusterd/859927/repl.t

index a500961..40e8602 100755

--- a/tests/bugs/glusterd/859927/repl.t

+++ b/tests/bugs/glusterd/859927/repl.t

@@ -23,7 +23,6 @@ TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2};

 TEST $CLI volume set $V0 cluster.self-heal-daemon off

 TEST $CLI volume set $V0 performance.stat-prefetch off

 TEST $CLI volume set $V0 client-log-level DEBUG

-TEST $CLI volume set $V0 cluster.background-self-heal-count 0

 TEST $CLI volume start $V0

 TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0;

@@ -34,6 +33,7 @@ EXPECT full volume_option $V0 cluster.data-self-heal-algorithm

 create_setup_for_self_heal $M0/a

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 0

 cat $file 2>&1 > /dev/null

+EXPECT_WITHIN $HEAL_TIMEOUT 0 get_pending_heal_count $V0

 TEST cmp $B0/${V0}1/a $B0/${V0}2/a

 TEST $CLI volume set $V0 cluster.data-self-heal-algorithm diff

@@ -41,12 +41,14 @@ EXPECT diff volume_option $V0 cluster.data-self-heal-algorithm

 create_setup_for_self_heal $M0/a

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 0

 cat $file 2>&1 > /dev/null

+EXPECT_WITHIN $HEAL_TIMEOUT 0 get_pending_heal_count $V0

 TEST cmp $B0/${V0}1/a $B0/${V0}2/a

 TEST $CLI volume reset $V0 cluster.data-self-heal-algorithm

 create_setup_for_self_heal $M0/a

 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 0

 cat $file 2>&1 > /dev/null

+EXPECT_WITHIN $HEAL_TIMEOUT 0 get_pending_heal_count $V0

 TEST cmp $B0/${V0}1/a $B0/${V0}2/a

 TEST ! $CLI volume set $V0 cluster.data-self-heal-algorithm ""

diff --git a/tests/bugs/quota/bug-1035576.t b/tests/bugs/quota/bug-1035576.t

index 99b3925..eaf4439 100644

--- a/tests/bugs/quota/bug-1035576.t

+++ b/tests/bugs/quota/bug-1035576.t

@@ -17,7 +17,6 @@ TEST $CLI volume set $V0 performance.io-cache off

 TEST $CLI volume set $V0 performance.write-behind off

 TEST $CLI volume set $V0 performance.stat-prefetch off

 TEST $CLI volume set $V0 performance.read-ahead off

-TEST $CLI volume set $V0 background-self-heal-count 0

 TEST $CLI volume set $V0 self-heal-daemon off

 TEST $CLI volume quota $V0 enable

diff --git a/tests/bugs/replicate/bug-802417.t b/tests/bugs/replicate/bug-802417.t

index df989b1..c5ba98b 100755

--- a/tests/bugs/replicate/bug-802417.t

+++ b/tests/bugs/replicate/bug-802417.t

@@ -32,7 +32,6 @@ TEST $CLI volume set $V0 performance.stat-prefetch off

 ## Make sure automatic self-heal doesn't perturb our results.

 TEST $CLI volume set $V0 cluster.self-heal-daemon off

 TEST $CLI volume set $V0 cluster.data-self-heal on

-TEST $CLI volume set $V0 cluster.background-self-heal-count 0

 ## Start volume and verify

 TEST $CLI volume start $V0;

@@ -70,8 +69,8 @@ tgt_xattr_2="trusted.afr.${V0}-client-2"

 actual=$(afr_get_changelog_xattr $obs_path_0 $tgt_xattr_0)

 EXPECT "0x000000000000000000000000|^\$" echo $actual

-actual=$(afr_get_changelog_xattr $obs_path_0 $tgt_xattr_1)

-EXPECT "0x000000000000000000000000|^\$" echo $actual

+EXPECT_WITHIN $HEAL_TIMEOUT "0x000000000000000000000000" \

+afr_get_changelog_xattr $obs_path_0 $tgt_xattr_1

 actual=$(afr_get_changelog_xattr $obs_path_0 $tgt_xattr_2)

 EXPECT "0x000000030000000000000000" echo $actual

diff --git a/tests/bugs/replicate/bug-977797.t b/tests/bugs/replicate/bug-977797.t

index 3ff14ec..72c616b 100755

--- a/tests/bugs/replicate/bug-977797.t

+++ b/tests/bugs/replicate/bug-977797.t

@@ -26,7 +26,6 @@ TEST $CLI volume set $V0 quick-read off

 TEST $CLI volume set $V0 read-ahead off

 TEST $CLI volume set $V0 write-behind off

 TEST $CLI volume set $V0 io-cache off

-TEST $CLI volume set $V0 background-self-heal-count 0

 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0

@@ -56,34 +55,29 @@ EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1;

 TEST dd if=$M0/a/file of=/dev/null bs=1024k

-b1c0dir=$(afr_get_specific_changelog_xattr $B0/$V0"1"/a \

-          trusted.afr.$V0-client-0 "entry")

-b1c1dir=$(afr_get_specific_changelog_xattr $B0/$V0"1"/a \

-          trusted.afr.$V0-client-1 "entry")

-b2c0dir=$(afr_get_specific_changelog_xattr \

-          $B0/$V0"2"/a trusted.afr.$V0-client-0 "entry")

-b2c1dir=$(afr_get_specific_changelog_xattr \

-          $B0/$V0"2"/a trusted.afr.$V0-client-1 "entry")

-

-

-b1c0f=$(afr_get_specific_changelog_xattr $B0/$V0"1"/a/file \

-        trusted.afr.$V0-client-0 "data")

-b1c1f=$(afr_get_specific_changelog_xattr $B0/$V0"1"/a/file \

-        trusted.afr.$V0-client-1 "data")

-b2c0f=$(afr_get_specific_changelog_xattr $B0/$V0"2"/a/file \

-        trusted.afr.$V0-client-0 "data")

-b2c1f=$(afr_get_specific_changelog_xattr $B0/$V0"2"/a/file \

-        trusted.afr.$V0-client-1 "data")

-

-EXPECT "00000000|^$" echo $b1c0f

-EXPECT "00000000|^$" echo $b1c1f

-EXPECT "00000000|^$" echo $b2c0f

-EXPECT "00000000|^$" echo $b2c1f

-

-EXPECT "00000000|^$" echo $b1c0dir

-EXPECT "00000000|^$" echo $b1c1dir

-EXPECT "00000000|^$" echo $b2c0dir

-EXPECT "00000000|^$" echo $b2c1dir

+EXPECT_WITHIN $HEAL_TIMEOUT "00000000"  \

+afr_get_specific_changelog_xattr $B0/$V0"1"/a/file trusted.afr.$V0-client-0 "data"

+

+EXPECT_WITHIN $HEAL_TIMEOUT "00000000" \

+afr_get_specific_changelog_xattr $B0/$V0"1"/a/file trusted.afr.$V0-client-1 "data"

+

+EXPECT_WITHIN $HEAL_TIMEOUT "00000000" \

+afr_get_specific_changelog_xattr $B0/$V0"2"/a/file trusted.afr.$V0-client-0 "data"

+

+EXPECT_WITHIN $HEAL_TIMEOUT  "00000000" \

+afr_get_specific_changelog_xattr $B0/$V0"2"/a/file trusted.afr.$V0-client-1 "data"

+

+EXPECT_WITHIN $HEAL_TIMEOUT "00000000" \

+afr_get_specific_changelog_xattr $B0/$V0"1"/a trusted.afr.$V0-client-0 "entry"

+

+EXPECT_WITHIN HEAL_TIMEOUT "00000000" \

+afr_get_specific_changelog_xattr $B0/$V0"1"/a trusted.afr.$V0-client-1 "entry"

+

+EXPECT_WITHIN $HEAL_TIMEOUT "00000000" \

+afr_get_specific_changelog_xattr $B0/$V0"2"/a trusted.afr.$V0-client-0 "entry"

+

+EXPECT_WITHIN $HEAL_TIMEOUT  "00000000" \

+afr_get_specific_changelog_xattr $B0/$V0"2"/a trusted.afr.$V0-client-1 "entry"

 ## Finish up

 TEST $CLI volume stop $V0;

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c

index b232a2d..b5d07ac 100644

--- a/xlators/cluster/afr/src/afr-common.c

+++ b/xlators/cluster/afr/src/afr-common.c

@@ -697,7 +697,8 @@ afr_accuse_smallfiles (xlator_t *this, struct afr_reply *replies,

 }

 int

-afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode)

+afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode,

+                       gf_boolean_t *start_heal)

 {

 	afr_local_t *local = NULL;

 	afr_private_t *priv = NULL;

@@ -777,6 +778,13 @@ afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode)

 		}

 	}

+	for (i = 0; i < priv->child_count; i++) {

+                if (start_heal && priv->child_up[i] &&

+                    (!data_readable[i] || !metadata_readable[i])) {

+                        *start_heal = _gf_true;

+                        break;

+                }

+        }

 	afr_inode_read_subvol_set (inode, this, data_readable,

 				   metadata_readable, event_generation);

 	return ret;

@@ -815,36 +823,6 @@ ret:

 	return -err;

 }

-

-int

-afr_refresh_selfheal_wrap (void *opaque)

-{

-	call_frame_t *frame = opaque;

-	afr_local_t *local = NULL;

-	xlator_t *this = NULL;

-	int err = 0;

-

-	local = frame->local;

-	this = frame->this;

-

-	afr_selfheal (frame->this, local->refreshinode->gfid);

-

-	afr_selfheal_unlocked_discover (frame, local->refreshinode,

-					local->refreshinode->gfid,

-					local->replies);

-

-	afr_replies_interpret (frame, this, local->refreshinode);

-

-	err = afr_inode_refresh_err (frame, this);

-

-        afr_local_replies_wipe (local, this->private);

-

-	local->refreshfn (frame, this, err);

-

-	return 0;

-}

-

-

 gf_boolean_t

 afr_selfheal_enabled (xlator_t *this)

 {

@@ -860,35 +838,43 @@ afr_selfheal_enabled (xlator_t *this)

 	return data || priv->metadata_self_heal || priv->entry_self_heal;

 }

-

 int

 afr_inode_refresh_done (call_frame_t *frame, xlator_t *this)

 {

-	call_frame_t *heal = NULL;

+	call_frame_t *heal_frame = NULL;

 	afr_local_t *local = NULL;

+        gf_boolean_t start_heal = _gf_false;

+        afr_local_t *heal_local = NULL;

+        int op_errno = ENOMEM;

 	int ret = 0;

 	int err = 0;

 	local = frame->local;

-	ret = afr_replies_interpret (frame, this, local->refreshinode);

+	ret = afr_replies_interpret (frame, this, local->refreshinode,

+                                     &start_heal);

 	err = afr_inode_refresh_err (frame, this);

         afr_local_replies_wipe (local, this->private);

-	if (ret && afr_selfheal_enabled (this)) {

-		heal = copy_frame (frame);

-		if (heal)

-			heal->root->pid = GF_CLIENT_PID_SELF_HEALD;

-		ret = synctask_new (this->ctx->env, afr_refresh_selfheal_wrap,

-				    afr_refresh_selfheal_done, heal, frame);

-		if (ret)

-			goto refresh_done;

-	} else {

-	refresh_done:

-		local->refreshfn (frame, this, err);

-	}

+	if (ret && afr_selfheal_enabled (this) && start_heal) {

+                heal_frame = copy_frame (frame);

+                if (!heal_frame)

+                        goto refresh_done;

+                heal_frame->root->pid = GF_CLIENT_PID_SELF_HEALD;

+                heal_local = AFR_FRAME_INIT (heal_frame, op_errno);

+                if (!heal_local) {

+                        AFR_STACK_DESTROY (heal_frame);

+                        goto refresh_done;

+                }

+                heal_local->refreshinode = inode_ref (local->refreshinode);

+                heal_local->heal_frame = heal_frame;

+                afr_throttled_selfheal (heal_frame, this);

+        }

+

+refresh_done:

+        local->refreshfn (frame, this, err);

 	return 0;

 }

@@ -1785,7 +1771,7 @@ afr_lookup_done (call_frame_t *frame, xlator_t *this)

 		*/

                 gf_uuid_copy (args.gfid, read_gfid);

                 args.ia_type = ia_type;

-		if (afr_replies_interpret (frame, this, local->inode)) {

+		if (afr_replies_interpret (frame, this, local->inode, NULL)) {

                         read_subvol = afr_read_subvol_decide (local->inode,

                                                               this, &args);

 			afr_inode_read_subvol_reset (local->inode, this);

@@ -2246,7 +2232,7 @@ afr_discover_done (call_frame_t *frame, xlator_t *this)

                 goto unwind;

 	}

-	afr_replies_interpret (frame, this, local->inode);

+	afr_replies_interpret (frame, this, local->inode, NULL);

 	read_subvol = afr_read_subvol_decide (local->inode, this, NULL);

 	if (read_subvol == -1) {

@@ -3899,6 +3885,12 @@ afr_priv_dump (xlator_t *this)

         gf_proc_dump_write("favorite_child", "%d", priv->favorite_child);

         gf_proc_dump_write("wait_count", "%u", priv->wait_count);

         gf_proc_dump_write("quorum-reads", "%d", priv->quorum_reads);

+        gf_proc_dump_write("heal-wait-queue-length", "%d",

+                           priv->heal_wait_qlen);

+        gf_proc_dump_write("heal-waiters", "%d", priv->heal_waiters);

+        gf_proc_dump_write("background-self-heal-count", "%d",

+                           priv->background_self_heal_count);

+        gf_proc_dump_write("healers", "%d", priv->healers);

         return 0;

 }

@@ -4205,6 +4197,7 @@ afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)

 		goto out;

 	}

+        INIT_LIST_HEAD (&local->healer);

 	return 0;

 out:

         return -1;

diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c

index 3d586dd..887298b 100644

--- a/xlators/cluster/afr/src/afr-dir-write.c

+++ b/xlators/cluster/afr/src/afr-dir-write.c

@@ -93,7 +93,7 @@ __afr_dir_write_finalize (call_frame_t *frame, xlator_t *this)

 	priv = this->private;

 	if (local->inode) {

-		afr_replies_interpret (frame, this, local->inode);

+		afr_replies_interpret (frame, this, local->inode, NULL);

 		inode_read_subvol = afr_data_subvol_get (local->inode, this,

 							 NULL, NULL, NULL);

 	}

diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c

index 6e90de0..73d7e94 100644

--- a/xlators/cluster/afr/src/afr-self-heal-common.c

+++ b/xlators/cluster/afr/src/afr-self-heal-common.c

@@ -20,6 +20,9 @@

 #include "protocol-common.h"

 #include "afr-messages.h"

+void

+afr_heal_synctask (xlator_t *this, afr_local_t *local);

+

 int

 afr_selfheal_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

 			  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)

@@ -1423,3 +1426,110 @@ afr_selfheal (xlator_t *this, uuid_t gfid)

 	return ret;

 }

+

+afr_local_t*

+__afr_dequeue_heals (afr_private_t *priv)

+{

+        afr_local_t *local = NULL;

+

+        if (list_empty (&priv->heal_waiting))

+                goto none;

+        if ((priv->background_self_heal_count > 0) &&

+            (priv->healers >= priv->background_self_heal_count))

+                goto none;

+

+        local = list_entry (priv->heal_waiting.next, afr_local_t, healer);

+        priv->heal_waiters--;

+        GF_ASSERT (priv->heal_waiters >= 0);

+        list_del_init(&local->healer);

+        list_add(&local->healer, &priv->healing);

+        priv->healers++;

+        return local;

+none:

+        gf_msg_debug (THIS->name, 0, "Nothing dequeued. "

+                      "Num healers: %d, Num Waiters: %d",

+                      priv->healers, priv->heal_waiters);

+        return NULL;

+}

+

+int

+afr_refresh_selfheal_wrap (void *opaque)

+{

+        call_frame_t *heal_frame = opaque;

+        afr_local_t *local = heal_frame->local;

+        int ret = 0;

+

+        ret = afr_selfheal (heal_frame->this, local->refreshinode->gfid);

+        return ret;

+}

+

+int

+afr_refresh_heal_done (int ret, call_frame_t *frame, void *opaque)

+{

+        call_frame_t *heal_frame = opaque;

+        xlator_t *this = heal_frame->this;

+        afr_private_t *priv = this->private;

+        afr_local_t *local = heal_frame->local;

+

+        LOCK (&priv->lock);

+        {

+                list_del_init(&local->healer);

+                priv->healers--;

+                GF_ASSERT (priv->healers >= 0);

+                local = __afr_dequeue_heals (priv);

+        }

+        UNLOCK (&priv->lock);

+

+        if (heal_frame)

+                AFR_STACK_DESTROY (heal_frame);

+

+        if (local)

+                afr_heal_synctask (this, local);

+        return 0;

+

+}

+

+void

+afr_heal_synctask (xlator_t *this, afr_local_t *local)

+{

+        int ret = 0;

+        call_frame_t *heal_frame = NULL;

+

+        heal_frame = local->heal_frame;

+        ret = synctask_new (this->ctx->env, afr_refresh_selfheal_wrap,

+                            afr_refresh_heal_done, heal_frame, heal_frame);

+        if (ret < 0)

+                /* Heal not launched. Will be queued when the next inode

+                 * refresh happens and shd hasn't healed it yet. */

+                afr_refresh_heal_done (ret, heal_frame, heal_frame);

+}

+

+void

+afr_throttled_selfheal (call_frame_t *frame, xlator_t *this)

+{

+        gf_boolean_t can_heal = _gf_true;

+        afr_private_t *priv = this->private;

+        afr_local_t *local = frame->local;

+

+        LOCK (&priv->lock);

+        {

+                if ((priv->background_self_heal_count > 0) &&

+                    (priv->heal_wait_qlen + priv->background_self_heal_count) >

+                    (priv->heal_waiters + priv->healers)) {

+                        list_add_tail(&local->healer, &priv->heal_waiting);

+                        priv->heal_waiters++;

+                        local = __afr_dequeue_heals (priv);

+                } else {

+                        can_heal = _gf_false;

+                }

+        }

+        UNLOCK (&priv->lock);

+

+        if (can_heal) {

+                if (local)

+                        afr_heal_synctask (this, local);

+                else

+                        gf_msg_debug (this->name, 0, "Max number of heals are "

+                                      "pending, background self-heal rejected.");

+        }

+}

diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h

index 74e852a..b298fa1 100644

--- a/xlators/cluster/afr/src/afr-self-heal.h

+++ b/xlators/cluster/afr/src/afr-self-heal.h

@@ -85,6 +85,9 @@

 int

 afr_selfheal (xlator_t *this, uuid_t gfid);

+void

+afr_throttled_selfheal (call_frame_t *frame, xlator_t *this);

+

 int

 afr_selfheal_name (xlator_t *this, uuid_t gfid, const char *name,

                    void *gfid_req);

diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c

index 5ef920a..d65895a 100644

--- a/xlators/cluster/afr/src/afr.c

+++ b/xlators/cluster/afr/src/afr.c

@@ -128,6 +128,10 @@ reconfigure (xlator_t *this, dict_t *options)

                           priv->background_self_heal_count, options, uint32,

                           out);

+        GF_OPTION_RECONF ("heal-wait-queue-length",

+                          priv->heal_wait_qlen, options, uint32, out);

+

+

         GF_OPTION_RECONF ("metadata-self-heal",

                           priv->metadata_self_heal, options, bool, out);

@@ -277,6 +281,8 @@ init (xlator_t *this)

         priv->read_child = -1;

         GF_OPTION_INIT ("arbiter-count", priv->arbiter_count, uint32, out);

+        INIT_LIST_HEAD (&priv->healing);

+        INIT_LIST_HEAD (&priv->heal_waiting);

         priv->spb_choice_timeout = AFR_DEFAULT_SPB_CHOICE_TIMEOUT;

@@ -329,6 +335,9 @@ init (xlator_t *this)

         GF_OPTION_INIT ("background-self-heal-count",

                         priv->background_self_heal_count, uint32, out);

+        GF_OPTION_INIT ("heal-wait-queue-length",

+                        priv->heal_wait_qlen, uint32, out);

+

         GF_OPTION_INIT ("data-self-heal", priv->data_self_heal, str, out);

         GF_OPTION_INIT ("data-self-heal-algorithm",

@@ -587,10 +596,21 @@ struct volume_options options[] = {

         { .key  = {"background-self-heal-count"},

           .type = GF_OPTION_TYPE_INT,

           .min  = 0,

-          .default_value = "16",

+          .max  = 256,

+          .default_value = "8",

+          .validate = GF_OPT_VALIDATE_MIN,

+          .description = "This specifies the number of per client self-heal "

+                         "jobs that can perform parallel heals in the "

+                         "background."

+        },

+        { .key  = {"heal-wait-queue-length"},

+          .type = GF_OPTION_TYPE_INT,

+          .min  = 0,

+          .max  = 10000, /*Around 100MB with sizeof(afr_local_t)= 10496 bytes*/

+          .default_value = "128",

           .validate = GF_OPT_VALIDATE_MIN,

-          .description = "This specifies the number of self-heals that can be "

-                         " performed in background without blocking the fop"

+          .description = "This specifies the number of heals that can be queued"

+                         " for the parallel background self heal jobs."

         },

         { .key  = {"data-self-heal"},

           .type = GF_OPTION_TYPE_STR,

diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h

index 52f9c51..9915344 100644

--- a/xlators/cluster/afr/src/afr.h

+++ b/xlators/cluster/afr/src/afr.h

@@ -74,8 +74,17 @@ typedef struct _afr_private {

         unsigned int data_self_heal_window_size;  /* max number of pipelined

                                                      read/writes */

-        unsigned int background_self_heal_count;

-        unsigned int background_self_heals_started;

+        struct list_head heal_waiting; /*queue for files that need heal*/

+        uint32_t  heal_wait_qlen; /*configurable queue length for heal_waiting*/

+        int32_t  heal_waiters; /* No. of elements currently in wait queue.*/

+

+        struct list_head healing;/* queue for files that are undergoing

+                                    background heal*/

+        uint32_t  background_self_heal_count;/*configurable queue length for

+                                               healing queue*/

+        int32_t  healers;/* No. of elements currently undergoing background

+                          heal*/

+

         gf_boolean_t metadata_self_heal;   /* on/off */

         gf_boolean_t entry_self_heal;      /* on/off */

@@ -127,12 +136,14 @@ typedef struct _afr_private {

 	afr_self_heald_t       shd;

-	/* pump dependencies */

-	void                   *pump_private;

-	gf_boolean_t           use_afr_in_pump;

         gf_boolean_t           consistent_metadata;

         uint64_t               spb_choice_timeout;

         gf_boolean_t           need_heal;

+

+	/* pump dependencies */

+	void                   *pump_private;

+	gf_boolean_t           use_afr_in_pump;

+

 } afr_private_t;

@@ -740,6 +751,10 @@ typedef struct _afr_local {

         int             xflag;

         gf_boolean_t    do_discovery;

 	struct afr_reply *replies;

+

+        /* For  client side background heals. */

+        struct list_head healer;

+        call_frame_t *heal_frame;

 } afr_local_t;

@@ -891,7 +906,8 @@ int

 afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);

 int

-afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode);

+afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode,

+                       gf_boolean_t *start_heal);

 void

 afr_local_replies_wipe (afr_local_t *local, afr_private_t *priv);

diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c

index f5746c8..1b68c1b 100644

--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c

+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c

@@ -1129,6 +1129,14 @@ struct volopt_map_entry glusterd_volopt_map[] = {

           .op_version = GD_OP_VERSION_RHS_3_0_4,

           .flags      = OPT_FLAG_CLIENT_OPT

         },

+        { .key        = "cluster.heal-wait-queue-length",

+          .voltype    = "cluster/replicate",

+          .type       = DOC,

+          .op_version = GD_OP_VERSION_3_7_10,

+          .flags      = OPT_FLAG_CLIENT_OPT

+        },

+

+        /* stripe xlator options */

         { .key         = "cluster.stripe-block-size",

           .voltype     = "cluster/stripe",

           .option      = "block-size",

-- 

1.7.1