From 8b3ff01d3fa96dddee783e840d331d788ab08d77 Mon Sep 17 00:00:00 2001

From: karthik-us <ksubrahm@redhat.com>

Date: Wed, 19 Apr 2017 18:04:46 +0530

Subject: [PATCH 459/473] cluster/afr: Return the list of node_uuids for the

 subvolume

Backport of: https://review.gluster.org/17084

Problem:

AFR was returning the node uuid of the first node for every file if

the replica set was healthy, which was resulting in only one node

migrating all the files.

Fix:

With this patch AFR returns the list of node_uuids to the upper layer,

so that they can decide on which node to migrate which files, resulting

in improved performance. Ordering of node uuids will be maintained based

on the ordering of the bricks. If a brick is down, then the node uuid

for that will be set to all zeros.

Change-Id: I2f2c13879c33c7b8baf392433c72686242360765

BUG: 1315781

Signed-off-by: karthik-us <ksubrahm@redhat.com>

Signed-off-by: Ravishankar N <ravishankar@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/106641

---

 xlators/cluster/afr/src/afr-common.c     |  49 +++++++++++

 xlators/cluster/afr/src/afr-inode-read.c | 141 ++++++++++++++++++++-----------

 xlators/cluster/afr/src/afr.h            |   5 ++

 3 files changed, 145 insertions(+), 50 deletions(-)

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c

index e9117b8..316cc40 100644

--- a/xlators/cluster/afr/src/afr-common.c

+++ b/xlators/cluster/afr/src/afr-common.c

@@ -5967,3 +5967,52 @@ mdata_unlock:

         return ret;

 }

+

+/*

+ * Concatenates the xattrs in local->replies separated by a delimiter.

+ */

+int

+afr_serialize_xattrs_with_delimiter (call_frame_t *frame, xlator_t *this,

+                                     char *buf, const char *default_str,

+                                     int32_t *serz_len, char delimiter)

+{

+        afr_private_t *priv      = NULL;

+        afr_local_t   *local     = NULL;

+        char          *xattr     = NULL;

+        int            i         = 0;

+        int            len       = 0;

+        int            ret       = -1;

+

+        priv = this->private;

+        local = frame->local;

+

+        for (i = 0; i < priv->child_count; i++) {

+                if (!local->replies[i].valid || local->replies[i].op_ret) {

+                        buf = strncat (buf, default_str, strlen (default_str));

+                        len += strlen (default_str);

+                        buf[len++] = delimiter;

+                        buf[len] = '\0';

+                } else {

+                        ret = dict_get_str (local->replies[i].xattr,

+                                            local->cont.getxattr.name, &xattr);

+                        if (ret) {

+                                gf_msg ("TEST", GF_LOG_ERROR, -ret,

+                                        AFR_MSG_DICT_GET_FAILED,

+                                        "Failed to get the node_uuid of brick "

+                                        "%d", i);

+                                goto out;

+                        }

+                        buf = strncat (buf, xattr, strlen (xattr));

+                        len += strlen (xattr);

+                        buf[len++] = delimiter;

+                        buf[len] = '\0';

+                }

+        }

+        buf[--len] = '\0'; /*remove the last delimiter*/

+        if (serz_len)

+                *serz_len = ++len;

+        ret = 0;

+

+out:

+        return ret;

+}

diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c

index 2b369ca..20446d8 100644

--- a/xlators/cluster/afr/src/afr-inode-read.c

+++ b/xlators/cluster/afr/src/afr-inode-read.c

@@ -713,57 +713,110 @@ unwind:

         return ret;

 }

+

 /**

- * node-uuid cbk uses next child querying mechanism

+ * node-uuid cbk returns the list of node_uuids for the subvolume.

  */

 int32_t

-afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,

-                            xlator_t *this, int32_t op_ret, int32_t op_errno,

-                            dict_t *dict, dict_t *xdata)

+afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

+                            int32_t op_ret, int32_t op_errno, dict_t *dict,

+                            dict_t *xdata)

 {

-        afr_private_t  *priv            = NULL;

-        afr_local_t    *local           = NULL;

-        xlator_t      **children        = NULL;

-        int             unwind          = 1;

-        int             curr_call_child = 0;

+        afr_local_t    *local          = NULL;

+        afr_private_t  *priv           = NULL;

+        int32_t         callcnt        = 0;

+        int             ret            = 0;

+        char           *xattr_serz     = NULL;

+        long            cky            = 0;

+        int32_t         tlen           = 0;

+        local = frame->local;

         priv = this->private;

-        children = priv->children;

+        cky = (long) cookie;

-        local = frame->local;

+        LOCK (&frame->lock);

+        {

+                callcnt = --local->call_count;

+                local->replies[cky].valid = 1;

+                local->replies[cky].op_ret = op_ret;

+                local->replies[cky].op_errno = op_errno;

-        if (op_ret == -1) { /** query the _next_ child */

+                if (op_ret < 0)

+                        goto unlock;

-                /**

-                 * _current_ becomes _next_

-                 * If done with all childs and yet no success; give up !

-                 */

-                curr_call_child = (int) ((long)cookie);

-                if (++curr_call_child == priv->child_count)

-                        goto unwind;

+                local->op_ret = 0;

-                gf_msg_debug (this->name, op_errno,

-                              "op_ret (-1): Re-querying afr-child (%d/%d)",

-                              curr_call_child, priv->child_count);

-

-                unwind = 0;

-                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,

-                                   (void *) (long) curr_call_child,

-                                   children[curr_call_child],

-                                   children[curr_call_child]->fops->getxattr,

-                                   &local->loc,

-                                   local->cont.getxattr.name,

-                                   NULL);

+                if (!local->xdata_rsp && xdata)

+                                local->xdata_rsp = dict_ref (xdata);

+                local->replies[cky].xattr = dict_ref (dict);

         }

- unwind:

-        if (unwind)

-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,

-                                  NULL);

+unlock:

+        UNLOCK (&frame->lock);

-        return 0;

+        if (!callcnt) {

+

+                if (local->op_ret != 0) {

+                        /* All bricks gave an error. */

+                        local->op_errno = afr_final_errno (local, priv);

+                        goto unwind;

+                }

+

+                /*Since we store the UUID0_STR as node uuid for down bricks and

+                 *for non zero op_ret, assigning length to  priv->child_count

+                 *number of uuids*/

+                local->cont.getxattr.xattr_len = (strlen (UUID0_STR) + 2) *

+                                                  priv->child_count;

+

+                if (!local->dict)

+                        local->dict = dict_new ();

+                if (!local->dict) {

+                        local->op_ret = -1;

+                        local->op_errno = ENOMEM;

+                        goto unwind;

+                }

+

+                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,

+                                        sizeof (char), gf_common_mt_char);

+

+                if (!xattr_serz) {

+                        local->op_ret = -1;

+                        local->op_errno = ENOMEM;

+                        goto unwind;

+                }

+

+                ret = afr_serialize_xattrs_with_delimiter (frame, this,

+                                                           xattr_serz,

+                                                           UUID0_STR, &tlen,

+                                                           ' ');

+                if (ret) {

+                        local->op_ret = -1;

+                        local->op_errno = ENOMEM;

+                        goto unwind;

+                }

+                ret = dict_set_dynstr (local->dict, local->cont.getxattr.name,

+                                       xattr_serz);

+                if (ret) {

+                        gf_msg (this->name, GF_LOG_ERROR,

+                                -ret, AFR_MSG_DICT_SET_FAILED,

+                                "Cannot set node_uuid key in dict");

+                        local->op_ret = -1;

+                        local->op_errno = ENOMEM;

+                } else {

+                        local->op_ret = local->cont.getxattr.xattr_len - 1;

+                        local->op_errno = 0;

+                }

+

+unwind:

+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,

+                                  local->op_errno, local->dict,

+                                  local->xdata_rsp);

+        }

+

+        return ret;

 }

+

 int32_t

 afr_getxattr_quota_size_cbk (call_frame_t *frame, void *cookie,

                              xlator_t *this, int32_t op_ret, int32_t op_errno,

@@ -1374,6 +1427,8 @@ afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk,

                 *cbk = afr_common_getxattr_stime_cbk;

         } else if (strcmp (name, QUOTA_SIZE_KEY) == 0) {

                 *cbk = afr_getxattr_quota_size_cbk;

+        } else if (!strcmp (name, GF_XATTR_NODE_UUID_KEY)) {

+                *cbk = afr_getxattr_node_uuid_cbk;

         } else {

                 is_spl = _gf_false;

         }

@@ -1489,9 +1544,7 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,

               loc_t *loc, const char *name, dict_t *xdata)

 {

         afr_private_t           *priv         = NULL;

-        xlator_t                **children    = NULL;

         afr_local_t             *local        = NULL;

-        int                     i             = 0;

         int32_t                 op_errno      = 0;

         int                     ret           = -1;

         fop_getxattr_cbk_t      cbk           = NULL;

@@ -1503,8 +1556,6 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,

         priv     = this->private;

-        children = priv->children;

-

         loc_copy (&local->loc, loc);

 	local->op = GF_FOP_GETXATTR;

@@ -1545,16 +1596,6 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,

                 return 0;

         }

-        if (XATTR_IS_NODE_UUID (name)) {

-                i = 0;

-                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,

-                                   (void *) (long) i,

-                                   children[i],

-                                   children[i]->fops->getxattr,

-                                   loc, name, xdata);

-                return 0;

-        }

-

 no_name:

 	afr_read_txn (frame, this, local->loc.inode, afr_getxattr_wind,

diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h

index 92b54f8..05f8249 100644

--- a/xlators/cluster/afr/src/afr.h

+++ b/xlators/cluster/afr/src/afr.h

@@ -1232,4 +1232,9 @@ afr_quorum_errno (afr_private_t *priv);

 gf_boolean_t

 afr_is_inode_refresh_reqd (inode_t *inode, xlator_t *this,

                            int event_gen1, int event_gen2);

+

+int

+afr_serialize_xattrs_with_delimiter (call_frame_t *frame, xlator_t *this,

+                                     char *buf, const char *default_str,

+                                     int32_t *serz_len, char delimiter);

 #endif /* __AFR_H__ */

-- 

1.8.3.1