From 27081fda822921e7f452304bea170d2d13cba257 Mon Sep 17 00:00:00 2001

From: Mohit Agrawal <moagrawa@redhat.com>

Date: Fri, 12 May 2017 21:12:47 +0530

Subject: [PATCH 205/212] cluster/dht : User xattrs are not healed after brick

 stop/start

Problem: In a distributed volume custom extended attribute value for a directory

         does not display correct value after stop/start or added newly brick.

         If any extended(acl) attribute value is set for a directory after stop/added

         the brick the attribute(user|acl|quota) value is not updated on brick

         after start the brick.

Solution: First store hashed subvol or subvol(has internal xattr) on inode ctx and

          consider it as a MDS subvol.At the time of update custom xattr

          (user,quota,acl, selinux) on directory first check the mds from

          inode ctx, if mds is not present on inode ctx then throw EINVAL error

          to application otherwise set xattr on MDS subvol with internal xattr

          value of -1 and then try to update the attribute on other non MDS

          volumes also.If mds subvol is down in that case throw an

          error "Transport endpoint is not connected". In dht_dir_lookup_cbk|

          dht_revalidate_cbk|dht_discover_complete call dht_call_dir_xattr_heal

          to heal custom extended attribute.

          In case of gnfs server if hashed subvol has not found based on

          loc then wind a call on all subvol to update xattr.

Fix:    1) Save MDS subvol on inode ctx

        2) Check if mds subvol is present on inode ctx

        3) If mds subvol is down then call unwind with error ENOTCONN and if it is up

           then set new xattr "GF_DHT_XATTR_MDS" to -1 and wind a call on other

           subvol.

        4) If setxattr fop is successful on non-mds subvol then increment the value of

           internal xattr to +1

        5) At the time of directory_lookup check the value of new xattr GF_DHT_XATTR_MDS

        6) If value is not 0 in dht_lookup_dir_cbk(other cbk) functions then call heal

           function to heal user xattr

        7) syncop_setxattr on hashed_subvol to reset the value of xattr to 0

           if heal is successful on all subvol.

Test : To reproduce the issue followed below steps

       1) Create a distributed volume and create mount point

       2) Create some directory from mount point mkdir tmp{1..5}

       3) Kill any one brick from the volume

       4) Set extended attribute from mount point on directory

          setfattr -n user.foo -v "abc" ./tmp{1..5}

          It will throw error " Transport End point is not connected "

          for those hashed subvol is down

       5) Start volume with force option to start brick process

       6) Execute getfattr command on mount point for directory

       7) Check extended attribute on brick

          getfattr -n user.foo <volume-location>/tmp{1..5}

          It shows correct value for directories for those

          xattr fop were executed successfully.

Note: The patch will resolve xattr healing problem only for fuse mount

      not for nfs mount.

> BUG: 1371806

> Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

> (Cherry pick from commit 9b4de61a136b8e5ba7bf0e48690cdb1292d0dee8)

> (Upstream patch link https://review.gluster.org/#/c/15468/)

BUG: 1550315

Change-Id: I4eb137eace24a8cb796712b742f1d177a65343d5

Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/132383

Tested-by: RHGS Build Bot <nigelb@redhat.com>

Reviewed-by: Raghavendra Gowdappa <rgowdapp@redhat.com>

Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>

---

 tests/bugs/bug-1368312.t                  |   30 +-

 tests/bugs/bug-1371806.t                  |   80 ++

 tests/bugs/bug-1371806_1.t                |   49 +

 tests/bugs/bug-1371806_2.t                |   52 ++

 tests/bugs/bug-1371806_3.t                |   63 ++

 tests/bugs/bug-1371806_acl.t              |   90 ++

 tests/bugs/distribute/bug-862967.t        |    7 +-

 xlators/cluster/dht/src/dht-common.c      | 1389 ++++++++++++++++++++++++++---

 xlators/cluster/dht/src/dht-common.h      |   72 +-

 xlators/cluster/dht/src/dht-helper.c      |   65 ++

 xlators/cluster/dht/src/dht-inode-write.c |  163 +++-

 xlators/cluster/dht/src/dht-messages.h    |   28 +-

 xlators/cluster/dht/src/dht-selfheal.c    |  519 ++++++++++-

 xlators/cluster/dht/src/dht-shared.c      |    2 +

 14 files changed, 2436 insertions(+), 173 deletions(-)

 create mode 100644 tests/bugs/bug-1371806.t

 create mode 100644 tests/bugs/bug-1371806_1.t

 create mode 100644 tests/bugs/bug-1371806_2.t

 create mode 100644 tests/bugs/bug-1371806_3.t

 create mode 100644 tests/bugs/bug-1371806_acl.t

diff --git a/tests/bugs/bug-1368312.t b/tests/bugs/bug-1368312.t

index 135048f..61e5606 100644

--- a/tests/bugs/bug-1368312.t

+++ b/tests/bugs/bug-1368312.t

@@ -29,46 +29,46 @@ TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0;

 TEST mkdir $M0/tmp1

 #Create metadata split-brain

-TEST kill_brick $V0 $H0 $B0/${V0}0

+TEST kill_brick $V0 $H0 $B0/${V0}2

 TEST chmod 666 $M0/tmp1

 TEST $CLI volume start $V0 force

-TEST kill_brick $V0 $H0 $B0/${V0}1

-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0

+TEST kill_brick $V0 $H0 $B0/${V0}3

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2

 TEST chmod 757 $M0/tmp1

 TEST $CLI volume start $V0 force

-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0

-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 3

 EXPECT 2 get_pending_heal_count $V0

-TEST kill_brick $V0 $H0 $B0/${V0}2

+TEST kill_brick $V0 $H0 $B0/${V0}4

 TEST chmod 755 $M0/tmp1

 TEST $CLI volume start $V0 force

-TEST kill_brick $V0 $H0 $B0/${V0}3

-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2

+TEST kill_brick $V0 $H0 $B0/${V0}5

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 4

 TEST chmod 766 $M0/tmp1

 TEST $CLI volume start $V0 force

-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2

-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 3

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 4

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 5

 EXPECT 4 get_pending_heal_count $V0

-TEST kill_brick $V0 $H0 $B0/${V0}4

+TEST kill_brick $V0 $H0 $B0/${V0}0

 TEST chmod 765 $M0/tmp1

 TEST $CLI volume start $V0 force

-TEST kill_brick $V0 $H0 $B0/${V0}5

-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 4

+TEST kill_brick $V0 $H0 $B0/${V0}1

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0

 TEST chmod 756 $M0/tmp1

 TEST $CLI volume start $V0 force

-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 4

-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 5

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0

+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1

 EXPECT 6 get_pending_heal_count $V0

diff --git a/tests/bugs/bug-1371806.t b/tests/bugs/bug-1371806.t

new file mode 100644

index 0000000..7dc1613

--- /dev/null

+++ b/tests/bugs/bug-1371806.t

@@ -0,0 +1,80 @@

+#!/bin/bash

+. $(dirname $0)/../include.rc

+. $(dirname $0)/../volume.rc

+. $(dirname $0)/../dht.rc

+cleanup;

+

+function get_getfattr {

+        local path=$1

+        echo `getfattr -n user.foo $path` | cut -f2 -d"=" | sed -e 's/^"//'  -e 's/"$//'

+}

+

+function set_fattr {

+        for i in `seq 1 10`

+        do

+                setfattr -n user.foo -v "newabc" ./tmp${i}

+                if [ "$?" = "0" ]

+                 then

+                    succ=$((succ+1))

+                else

+                    fail=$((fail+1))

+                fi

+        done

+}

+

+

+

+TEST glusterd

+TEST pidof glusterd

+TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1,2,3,4,5}

+TEST $CLI volume start $V0

+

+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0;

+

+cd $M0

+TEST mkdir tmp{1..10}

+

+##First set user.foo xattr with value abc on all dirs

+

+TEST setfattr -n user.foo -v "abc" ./tmp{1..10}

+EXPECT "abc" get_getfattr ./tmp{1..10}

+EXPECT "abc" get_getfattr $B0/${V0}5/tmp{1..10}

+

+TEST kill_brick $V0 $H0 $B0/${V0}5

+EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "5" online_brick_count

+

+succ=fail=0

+## set user.foo xattr with value newabc after kill one brick

+set_fattr

+TEST $CLI volume start $V0 force

+EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "6" online_brick_count

+

+cd -

+TEST umount $M0

+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0;

+

+cd $M0

+## At this point dht code will heal xattr on down brick only for those dirs

+## hashed subvol was up at the time of update xattr

+TEST stat ./tmp{1..10}

+

+## Count the user.foo xattr value with abc on mount point and compare with fail value

+count=`getfattr -n user.foo ./tmp{1..10} | grep "user.foo" | grep -iw "abc" | wc -l`

+EXPECT "$fail" echo $count

+

+## Count the user.foo xattr value with newabc on mount point and compare with succ value

+count=`getfattr -n user.foo ./tmp{1..10} | grep "user.foo" | grep -iw "newabc" | wc -l`

+EXPECT "$succ" echo $count

+

+## Count the user.foo xattr value with abc on brick and compare with succ value

+count=`getfattr -n user.foo $B0/${V0}5/tmp{1..10} | grep "user.foo" | grep -iw "abc" | wc -l`

+EXPECT "$fail" echo $count

+

+## Count the user.foo xattr value with newabc on brick and compare with succ value

+count=`getfattr -n user.foo $B0/${V0}5/tmp{1..10} | grep "user.foo" | grep -iw "newabc" | wc -l`

+EXPECT "$succ" echo $count

+

+

+cd -

+cleanup

+exit

diff --git a/tests/bugs/bug-1371806_1.t b/tests/bugs/bug-1371806_1.t

new file mode 100644

index 0000000..44a57a9

--- /dev/null

+++ b/tests/bugs/bug-1371806_1.t

@@ -0,0 +1,49 @@

+#!/bin/bash

+. $(dirname $0)/../include.rc

+. $(dirname $0)/../volume.rc

+. $(dirname $0)/../dht.rc

+cleanup;

+

+function get_getfattr {

+        local path=$1

+        echo `getfattr -n user.foo $path` | cut -f2 -d"=" | sed -e 's/^"//'  -e 's/"$//'

+}

+

+function remove_mds_xattr {

+

+       for i in `seq 1 10`

+       do

+               setfattr -x trusted.glusterfs.dht.mds $1/tmp${i} 2> /dev/null

+       done

+}

+

+

+

+TEST glusterd

+TEST pidof glusterd

+TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1,2,3}

+TEST $CLI volume start $V0

+

+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0;

+

+cd $M0

+TEST mkdir tmp{1..10}

+

+##Remove internal mds xattr from all directory

+remove_mds_xattr $B0/${V0}0

+remove_mds_xattr $B0/${V0}1

+remove_mds_xattr $B0/${V0}2

+remove_mds_xattr $B0/${V0}3

+

+cd -

+umount $M0

+

+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0;

+cd $M0

+

+TEST setfattr -n user.foo -v "abc" ./tmp{1..10}

+EXPECT "abc" get_getfattr ./tmp{1..10}

+

+cd -

+cleanup

+exit

diff --git a/tests/bugs/bug-1371806_2.t b/tests/bugs/bug-1371806_2.t

new file mode 100644

index 0000000..e6aa8e7

--- /dev/null

+++ b/tests/bugs/bug-1371806_2.t

@@ -0,0 +1,52 @@

+#!/bin/bash

+. $(dirname $0)/../include.rc

+. $(dirname $0)/../volume.rc

+. $(dirname $0)/../dht.rc

+cleanup;

+

+function get_getfattr {

+        local path=$1

+        echo `getfattr -n user.foo $path` | cut -f2 -d"=" | sed -e 's/^"//'  -e 's/"$//'

+}

+

+function remove_mds_xattr {

+

+       for i in `seq 1 10`

+       do

+               setfattr -x trusted.glusterfs.dht.mds $1/tmp${i} 2> /dev/null

+       done

+}

+

+

+

+

+TEST glusterd

+TEST pidof glusterd

+TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1,2,3}

+TEST $CLI volume start $V0

+

+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 --attribute-timeout=0 $M0;

+cd $M0

+TEST mkdir tmp{1..10}

+

+##Remove internal mds xattr from all directory

+remove_mds_xattr $B0/${V0}0

+remove_mds_xattr $B0/${V0}1

+remove_mds_xattr $B0/${V0}2

+remove_mds_xattr $B0/${V0}3

+

+##First set user.foo xattr with value abc on all dirs

+

+TEST setfattr -n user.foo -v "abc" ./tmp{1..10}

+EXPECT "abc" get_getfattr ./tmp{1..10}

+EXPECT "abc" get_getfattr $B0/${V0}0/tmp{1..10}

+EXPECT "abc" get_getfattr $B0/${V0}1/tmp{1..10}

+EXPECT "abc" get_getfattr $B0/${V0}2/tmp{1..10}

+EXPECT "abc" get_getfattr $B0/${V0}3/tmp{1..10}

+

+cd -

+TEST umount $M0

+

+cd -

+cleanup

+exit

diff --git a/tests/bugs/bug-1371806_3.t b/tests/bugs/bug-1371806_3.t

new file mode 100644

index 0000000..cb13f37

--- /dev/null

+++ b/tests/bugs/bug-1371806_3.t

@@ -0,0 +1,63 @@

+#!/bin/bash

+. $(dirname $0)/../include.rc

+. $(dirname $0)/../volume.rc

+. $(dirname $0)/../dht.rc

+cleanup;

+

+function get_getfattr {

+        local path=$1

+        echo `getfattr -n user.foo $path` | cut -f2 -d"=" | sed -e 's/^"//'  -e 's/"$//'

+}

+

+function set_fattr {

+        for i in `seq 1 10`

+        do

+                setfattr -n user.foo -v "newabc" ./tmp${i}

+                if [ "$?" = "0" ]

+                 then

+                    succ=$((succ+1))

+                else

+                    fail=$((fail+1))

+                fi

+        done

+}

+

+

+

+TEST glusterd

+TEST pidof glusterd

+TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1,2,3}

+TEST $CLI volume start $V0

+

+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 --attribute-timeout=0 $M0;

+

+cd $M0

+TEST mkdir tmp{1..10}

+

+TEST kill_brick $V0 $H0 $B0/${V0}3

+EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "3" online_brick_count

+

+succ=fail=0

+## set user.foo xattr with value newabc after kill one brick

+set_fattr

+TEST $CLI volume start $V0 force

+EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "4" online_brick_count

+

+cd -

+TEST umount $M0

+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 --attribute-timeout=0 $M0;

+

+cd $M0

+## At this point dht code will heal xattr on down brick only for those dirs

+## hashed subvol was up at the time of update xattr

+TEST stat ./tmp{1..10}

+

+

+## Count the user.foo xattr value with newabc on brick and compare with succ value

+count=`getfattr -n user.foo $B0/${V0}3/tmp{1..10} | grep "user.foo" | grep -iw "newabc" | wc -l`

+EXPECT "$succ" echo $count

+

+

+cd -

+cleanup

+exit

diff --git a/tests/bugs/bug-1371806_acl.t b/tests/bugs/bug-1371806_acl.t

new file mode 100644

index 0000000..aa41e04

--- /dev/null

+++ b/tests/bugs/bug-1371806_acl.t

@@ -0,0 +1,90 @@

+#!/bin/bash

+. $(dirname $0)/../include.rc

+. $(dirname $0)/../volume.rc

+

+cleanup;

+TEST useradd tmpuser

+

+function set_facl_user {

+        for i in `seq 1 10`

+        do

+                setfacl -m u:tmpuser:rw ./tmp${i}

+                if [ "$?" = "0" ]

+                 then

+                    succ=$((succ+1))

+                else

+                    fail=$((fail+1))

+                fi

+        done

+}

+

+function set_facl_default {

+        for i in `seq 1 10`

+        do

+                setfacl -m d:o:rw ./tmp${i}

+                if [ "$?" = "0" ]

+                 then

+                    succ1=$((succ1+1))

+                else

+                    fail1=$((fail1+1))

+                fi

+        done

+}

+

+

+

+

+TEST glusterd

+TEST pidof glusterd

+TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1,2,3,4,5}

+TEST $CLI volume set $V0 diagnostics.client-log-level DEBUG

+TEST $CLI volume start $V0

+

+TEST glusterfs --volfile-id=$V0 --acl --volfile-server=$H0 --entry-timeout=0 $M0;

+

+cd $M0

+TEST mkdir tmp{1..10}

+TEST setfacl -m u:tmpuser:rwx ./tmp{1..10}

+count=`getfacl -p $M0/tmp{1..10} | grep -c "user:tmpuser:rwx"`

+EXPECT "10" echo $count

+TEST setfacl -m d:o:rwx ./tmp{1..10}

+count=`getfacl -p $M0/tmp{1..10} | grep -c "default:other::rwx"`

+EXPECT "10" echo $count

+count=`getfacl -p $B0/${V0}5/tmp{1..10} | grep -c "user:tmpuser:rwx"`

+EXPECT "10" echo $count

+count=`getfacl -p $B0/${V0}5/tmp{1..10} | grep -c "default:other::rwx"`

+EXPECT "10" echo $count

+

+

+TEST kill_brick $V0 $H0 $B0/${V0}5

+EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "5" online_brick_count

+

+succ=fail=0

+## Update acl attributes on dir after kill one brick

+set_facl_user

+succ1=fail1=0

+set_facl_default

+

+TEST $CLI volume start $V0 force

+EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "6" online_brick_count

+

+cd -

+TEST umount $M0

+TEST glusterfs --volfile-id=$V0 --acl --volfile-server=$H0 --entry-timeout=0 $M0;

+

+cd $M0

+## At this point dht will heal xatts on down brick only for those hashed_subvol

+## was up at the time of updated xattrs

+TEST stat ./tmp{1..10}

+

+## Compare succ value with updated acl attributes

+count=`getfacl -p $B0/${V0}5/tmp{1..10} | grep -c "user:tmpuser:rw-"`

+EXPECT "$succ" echo $count

+

+

+count=`getfacl -p $B0/${V0}5/tmp{1..10} | grep -c "default:other::rw-"`

+EXPECT "$succ1" echo $count

+

+cd -

+userdel --force tmpuser

+cleanup

diff --git a/tests/bugs/distribute/bug-862967.t b/tests/bugs/distribute/bug-862967.t

index 09dac37..2fb0848 100644

--- a/tests/bugs/distribute/bug-862967.t

+++ b/tests/bugs/distribute/bug-862967.t

@@ -37,7 +37,7 @@ chown 1:1 $M0/dir;

 # Kill a brick process

-kill_brick $V0 $H0 $B0/${V0}1

+kill_brick $V0 $H0 $B0/${V0}2

 # change dir ownership

 NEW_UID=36;

 NEW_GID=36;

@@ -51,9 +51,8 @@ sleep 10;

 ls -l $M0/dir;

 # check if uid/gid is healed on backend brick which was taken down

-BACKEND_UID=`stat -c %u $B0/${V0}1/dir`;

-BACKEND_GID=`stat -c %g $B0/${V0}1/dir`;

-

+BACKEND_UID=`stat -c %u $B0/${V0}2/dir`;

+BACKEND_GID=`stat -c %g $B0/${V0}2/dir`;

 EXPECT "0" uid_gid_compare $NEW_UID $NEW_GID $BACKEND_UID $BACKEND_GID

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c

index 5641330..f1e6a92 100644

--- a/xlators/cluster/dht/src/dht-common.c

+++ b/xlators/cluster/dht/src/dht-common.c

@@ -110,6 +110,24 @@ int dht_read_iatt_from_xdata (xlator_t *this, dict_t *xdata,

 int

 dht_rmdir_unlock (call_frame_t *frame, xlator_t *this);

+char *xattrs_to_heal[] = {

+        "user.",

+        POSIX_ACL_ACCESS_XATTR,

+        POSIX_ACL_DEFAULT_XATTR,

+        QUOTA_LIMIT_KEY,

+        QUOTA_LIMIT_OBJECTS_KEY,

+        GF_SELINUX_XATTR_KEY,

+        NULL

+};

+

+/* Return true if key exists in array

+*/

+static gf_boolean_t

+dht_match_xattr (const char *key)

+{

+        return gf_get_index_by_elem (xattrs_to_heal, (char *)key) >= 0;

+}

+

 int

 dht_aggregate_quota_xattr (dict_t *dst, char *key, data_t *value)

 {

@@ -205,7 +223,7 @@ int add_opt(char **optsp, const char *opt)

 }

 /* Return Choice list from Split brain status */

-char *

+static char *

 getChoices (const char *value)

 {

         int i = 0;

@@ -428,6 +446,74 @@ out:

         return;

 }

+/* Code to save hashed subvol on inode ctx as a mds subvol

+*/

+int

+dht_inode_ctx_mdsvol_set (inode_t *inode, xlator_t *this, xlator_t *mds_subvol)

+{

+        dht_inode_ctx_t         *ctx            = NULL;

+        int                      ret            = -1;

+        uint64_t                 ctx_int        = 0;

+        gf_boolean_t             ctx_free       = _gf_false;

+

+

+        LOCK (&inode->lock);

+        {

+                ret = __inode_ctx_get (inode, this , &ctx_int);

+                if (ctx_int) {

+                        ctx = (dht_inode_ctx_t *)ctx_int;

+                        ctx->mds_subvol = mds_subvol;

+                } else {

+                        ctx = GF_CALLOC (1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);

+                        if (!ctx)

+                                goto unlock;

+                        ctx->mds_subvol = mds_subvol;

+                        ctx_free        = _gf_true;

+                        ctx_int = (long) ctx;

+                        ret =  __inode_ctx_set (inode, this, &ctx_int);

+                }

+        }

+unlock:

+        UNLOCK (&inode->lock);

+        if (ret && ctx_free)

+                GF_FREE (ctx);

+        return ret;

+}

+

+/*Code to get mds subvol from inode ctx */

+

+int

+dht_inode_ctx_mdsvol_get (inode_t *inode, xlator_t *this, xlator_t **mdsvol)

+{

+        dht_inode_ctx_t         *ctx            = NULL;

+        int                      ret            = -1;

+

+        if (!mdsvol)

+                return ret;

+

+        if (__is_root_gfid(inode->gfid)) {

+                (*mdsvol) = FIRST_CHILD (this);

+                return 0;

+        }

+

+        ret = dht_inode_ctx_get (inode, this, &ctx;;

+

+        if (!ret && ctx) {

+                if (ctx->mds_subvol) {

+                        *mdsvol = ctx->mds_subvol;

+                        ret = 0;

+                } else {

+                        ret = -1;

+                }

+        }

+

+        return ret;

+}

+

+

+

+

+

 /* TODO:

    - use volumename in xattr instead of "dht"

    - use NS locks

@@ -443,6 +529,7 @@ dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,

 {

         dht_local_t  *local = NULL;

         dht_layout_t *layout = NULL;

+        dht_conf_t   *conf   = NULL;

         int           ret = -1;

         GF_VALIDATE_OR_GOTO ("dht", frame, out);

@@ -450,6 +537,7 @@ dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,

         GF_VALIDATE_OR_GOTO ("dht", frame->local, out);

         local = frame->local;

+        conf  = this->private;

         ret = op_ret;

         FRAME_SU_UNDO (frame, dht_local_t);

@@ -467,6 +555,8 @@ dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,

         DHT_STRIP_PHASE1_FLAGS (&local->stbuf);

         dht_set_fixed_dir_stat (&local->postparent);

+        /* Delete mds xattr at the time of STACK UNWIND */

+        GF_REMOVE_INTERNAL_XATTR (conf->mds_xattr_key, local->xattr);

         DHT_STACK_UNWIND (lookup, frame, ret, local->op_errno, local->inode,

                           &local->stbuf, local->xattr, &local->postparent);

@@ -492,10 +582,12 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)

         int              i               = 0;

         loc_t            loc             = {0 };

         int8_t           is_read_only    = 0, layout_anomalies = 0;

+        char             gfid_local[GF_UUID_BUF_SIZE] = {0};

         local = discover_frame->local;

         layout = local->layout;

         conf = this->private;

+        gf_uuid_unparse(local->gfid, gfid_local);

         LOCK(&discover_frame->lock);

         {

@@ -507,6 +599,18 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)

         if (!main_frame)

                 return 0;

+        /* Code to update all extended attributed from

+           subvol to local->xattr on that internal xattr has found

+        */

+        if (conf->subvolume_cnt == 1)

+                local->need_xattr_heal = 0;

+        if (local->need_xattr_heal && (local->mds_xattr)) {

+                dht_dir_set_heal_xattr (this, local, local->xattr,

+                                        local->mds_xattr, NULL, NULL);

+                dict_unref (local->mds_xattr);

+                local->mds_xattr = NULL;

+        }

+

         ret = dict_get_int8 (local->xattr_req, QUOTA_READ_ONLY_KEY,

                              &is_read_only);

         if (ret < 0)

@@ -575,6 +679,26 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)

                 }