From e51e7575c8194fd9f98d1c26dd6395030d95e1b2 Mon Sep 17 00:00:00 2001

From: Susant Palai <spalai@redhat.com>

Date: Tue, 10 Jan 2017 16:11:50 +0530

Subject: [PATCH 431/473] cluster/dht: rebalance perf enhancement

Problem: Throttle settings "normal" and "aggressive" for rebalance

did not have performance difference.

normal mode spawns $(no. of cores - 4)/2 threads and aggressive

spawns $(no. of cores - 4) threads. Though aggressive mode has twice

the number of threads compared to that of normal mode, there was no

performance gain when switched to aggressive mode from normal mode.

RCA:

During the course of debugging the above problem, we tried assigning

migration job to migration threads spawned by rebalance, rather than

synctasks(as there is more overhead associated to manage the task

queue and threads). This gave us a significant improvement over rebalance

under synctasks. This patch does not really gurantee that there will be a

clear performance difference between normal and aggressive mode, but this

patch certainly maximized the disk utilization for 1GBfiles run.

Results:

Test enviroment:

Gluster Config:

Number of Bricks: 2 (one brick per disk(RAID-6 12 disk))

Bricks:

Brick1: server1:/brick/test1/1

Brick2: server2:/brick/test1/1

Options Reconfigured:

performance.readdir-ahead: on

server.event-threads: 4

client.event-threads: 4

1000 files with 1GB each were created/renamed such that all files will have

server1 as cached and server2 as hashed, so that all files will be migrated.

Test machines had 24 cores each.

Results  with/without synctask based migration:

-----------------------------------------------

mode                    normal(10threads)          aggressive(20threads)

timetaken               0:55:30 (h:m:s)            0:56:3 (h:m:s)

withsynctask

timetaken

with migrator           0:38:3 (h:m:s)             0:23:41 (h:m:s)

threads

From above table it can be seen that, there is a clear 2x perf gain between

rebalance with synctask vs rebalance with migrator threads.

Additionally this patch modifies the code so that caller will have the exact error

number returned by dht_migrate_file(earlier the errno meaning was overloaded). This

will help avoiding scenarios where migration failure due to ENOENT, can result in

rebalance abort/failure.

> Change-Id: I8904e2fb147419d4a51c1267be11a08ffd52168e

> BUG: 1420166

> Signed-off-by: Susant Palai <spalai@redhat.com>

> Reviewed-on: https://review.gluster.org/16427

> Smoke: Gluster Build System <jenkins@build.gluster.org>

> Reviewed-by: N Balachandran <nbalacha@redhat.com>

> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

Change-Id: I8904e2fb147419d4a51c1267be11a08ffd52168e

BUG: 1381142

Signed-off-by: Susant Palai <spalai@redhat.com>

Reviewed-on: https://review.gluster.org/16427

Smoke: Gluster Build System <jenkins@build.gluster.org>

Reviewed-by: N Balachandran <nbalacha@redhat.com>

Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

Signed-off-by: Susant Palai <spalai@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/104898

Reviewed-by: Nithya Balachandran <nbalacha@redhat.com>

---

 libglusterfs/src/syncop.c               |   2 +-

 xlators/cluster/dht/src/dht-common.h    |   4 +-

 xlators/cluster/dht/src/dht-rebalance.c | 380 +++++++++++++++++++++++---------

 3 files changed, 280 insertions(+), 106 deletions(-)

diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c

index 00a9b57..badb056 100644

--- a/libglusterfs/src/syncop.c

+++ b/libglusterfs/src/syncop.c

@@ -1566,7 +1566,7 @@ syncop_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

 int

 syncop_setxattr (xlator_t *subvol, loc_t *loc, dict_t *dict, int32_t flags,

-		 dict_t *xdata_in, dict_t **xdata_out)

+                 dict_t *xdata_in, dict_t **xdata_out)

 {

         struct syncargs args = {0, };

diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h

index eb6d1e8..16cc056 100644

--- a/xlators/cluster/dht/src/dht-common.h

+++ b/xlators/cluster/dht/src/dht-common.h

@@ -1088,10 +1088,10 @@ gf_defrag_start (void *this);

 int32_t

 gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

-                           struct iatt *stbuf);

+                           struct iatt *stbuf, int *fop_errno);

 int

 dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,

-                 int flag);

+                 int flag, int *fop_errno);

 int

 dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this,

                           dht_layout_t **layout_int);

diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c

index b2279d0..9a7d9ab 100644

--- a/xlators/cluster/dht/src/dht-rebalance.c

+++ b/xlators/cluster/dht/src/dht-rebalance.c

@@ -18,12 +18,12 @@

 #include <signal.h>

 #include "events.h"

-

 #define GF_DISK_SECTOR_SIZE             512

 #define DHT_REBALANCE_PID               4242 /* Change it if required */

 #define DHT_REBALANCE_BLKSIZE           (1024 * 1024)  /* 1 MB */

 #define MAX_MIGRATE_QUEUE_COUNT         500

 #define MIN_MIGRATE_QUEUE_COUNT         200

+#define MAX_REBAL_TYPE_SIZE             16

 #ifndef MAX

 #define MAX(a, b) (((a) > (b))?(a):(b))

@@ -165,7 +165,8 @@ dht_send_rebalance_event (xlator_t *this, int cmd, gf_defrag_status_t status)

 static int

 dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,

-                      int32_t size, off_t offset, struct iobref *iobref)

+                      int32_t size, off_t offset, struct iobref *iobref,

+                      int *fop_errno)

 {

         int i            = 0;

         int ret          = -1;

@@ -199,6 +200,7 @@ dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,

                                         gf_log (THIS->name, GF_LOG_WARNING,

                                                 "failed to write (%s)",

                                                 strerror (-ret));

+                                        *fop_errno = -ret;

                                         ret = -1;

                                         goto out;

                                 }

@@ -219,6 +221,7 @@ dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,

                                 gf_log (THIS->name, GF_LOG_WARNING,

                                         "failed to write (%s)",

                                         strerror (-ret));

+                                *fop_errno = -ret;

                                 ret = -1;

                                 goto out;

                         }

@@ -272,7 +275,7 @@ be converted to "0" in dht_migrate_file.

 int32_t

 gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

-                           struct iatt *stbuf)

+                           struct iatt *stbuf, int *fop_errno)

 {

         int32_t                 ret             = -1;

         xlator_t               *cached_subvol   = NULL;

@@ -285,6 +288,9 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

         gf_loglevel_t          loglevel         = 0;

         dict_t                 *link_xattr      = NULL;

+

+        *fop_errno = EINVAL;

+

         GF_VALIDATE_OR_GOTO ("defrag", loc, out);

         GF_VALIDATE_OR_GOTO ("defrag", loc->name, out);

         GF_VALIDATE_OR_GOTO ("defrag", stbuf, out);

@@ -299,6 +305,8 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "Migrate file failed :"

                         "loc->pargfid is NULL for %s", loc->path);

+                *fop_errno = EINVAL;

+                ret = -1;

                 goto out;

         }

@@ -307,13 +315,15 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "Migrate file failed :"

                         "loc->gfid is NULL for %s", loc->path);

+                *fop_errno = EINVAL;

+                ret = -1;

                 goto out;

         }

         link_xattr = dict_new ();

         if (!link_xattr) {

                 ret = -1;

-                errno = ENOMEM;

+                *fop_errno = ENOMEM;

                 goto out;

         }

@@ -354,6 +364,7 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "Migrate file failed:%s lookup failed with ret = %d",

                         loc->path, ret);

+                *fop_errno = -ret;

                 ret = -1;

                 goto out;

         }

@@ -365,6 +376,8 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

                         "Migrate file failed :"

                         "Failed to get cached subvol"

                         " for %s on %s", loc->name, this->name);

+                *fop_errno = EINVAL;

+                ret = -1;

                 goto out;

         }

@@ -375,6 +388,8 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

                         "Migrate file failed :"

                         "Failed to get hashed subvol"

                         " for %s on %s", loc->name, this->name);

+                *fop_errno = EINVAL;

+                ret = -1;

                 goto out;

         }

@@ -398,6 +413,8 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

                                 "Failed to set dictionary value:"

                                 " key = %s for %s",

                                 conf->link_xattr_name, loc->name);

+                        *fop_errno = ENOMEM;

+                        ret = -1;

                         goto out;

                 }

@@ -410,6 +427,7 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

                                 "Linkto setxattr failed %s -> %s (%s)",

                                 cached_subvol->name,

                                 loc->name, strerror (-ret));

+                        *fop_errno = -ret;

                         ret = -1;

                         goto out;

                 }

@@ -439,8 +457,10 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

                                 " failed on  subvol %s", loc->name,

                                 uuid_utoa(loc->gfid),

                                 hashed_subvol->name);

-                        if (op_errno != EEXIST)

+                        if (op_errno != EEXIST) {

+                                *fop_errno = op_errno;

                                 goto out;

+                        }

                 }

         }

         ret = syncop_lookup (hashed_subvol, loc, &iatt, NULL, NULL, NULL);

@@ -450,15 +470,18 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,

                         "Migrate file failed :Failed lookup %s on %s ",

                         loc->name, hashed_subvol->name);

+                *fop_errno = -ret;

                 ret = -1;

                 goto out;

         }

         if (iatt.ia_nlink == stbuf->ia_nlink) {

                 ret = dht_migrate_file (this, loc, cached_subvol, hashed_subvol,

-                                        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS);

-                if (ret)

+                                        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS,

+                                        fop_errno);

+                if (ret) {

                         goto out;

+                }

         }

         ret = -2;

 out:

@@ -470,8 +493,8 @@ out:

 static int

 __check_file_has_hardlink (xlator_t *this, loc_t *loc,

-                      struct iatt *stbuf, dict_t *xattrs, int flags,

-                                gf_defrag_info_t *defrag)

+                           struct iatt *stbuf, dict_t *xattrs, int flags,

+                           gf_defrag_info_t *defrag, int *fop_errno)

 {

        int ret = 0;

@@ -484,7 +507,7 @@ __check_file_has_hardlink (xlator_t *this, loc_t *loc,

                 if (flags == GF_DHT_MIGRATE_HARDLINK) {

                         synclock_lock (&defrag->link_lock);

                         ret = gf_defrag_handle_hardlink

-                                (this, loc, xattrs, stbuf);

+                                (this, loc, xattrs, stbuf, fop_errno);

                         synclock_unlock (&defrag->link_lock);

                         /*

                           Returning zero will force the file to be remigrated.

@@ -501,9 +524,10 @@ __check_file_has_hardlink (xlator_t *this, loc_t *loc,

                 } else {

                         gf_msg (this->name, GF_LOG_WARNING, 0,

                                 DHT_MSG_MIGRATE_FILE_FAILED,

-                                "Migrate file failed:"

+                                "Migration skipped for:"

                                 "%s: file has hardlinks", loc->path);

-                        ret = -ENOTSUP;

+                        *fop_errno = ENOTSUP;

+                        ret = -1;

                 }

        }

@@ -522,7 +546,7 @@ __check_file_has_hardlink (xlator_t *this, loc_t *loc,

 static int

 __is_file_migratable (xlator_t *this, loc_t *loc,

                       struct iatt *stbuf, dict_t *xattrs, int flags,

-                                gf_defrag_info_t *defrag)

+                                gf_defrag_info_t *defrag, int *fop_errno)

 {

         int ret = -1;

         int lock_count = 0;

@@ -532,6 +556,7 @@ __is_file_migratable (xlator_t *this, loc_t *loc,

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "Migrate file failed:"

                         "%s: migrate-file called on directory", loc->path);

+                *fop_errno = EISDIR;

                 ret = -1;

                 goto out;

         }

@@ -545,6 +570,7 @@ __is_file_migratable (xlator_t *this, loc_t *loc,

                                 "Migrate file failed:"

                                 "%s: Unable to get lock count for file",

                                 loc->path);

+                        *fop_errno = EINVAL;

                         ret = -1;

                         goto out;

                 }

@@ -554,6 +580,7 @@ __is_file_migratable (xlator_t *this, loc_t *loc,

                                 DHT_MSG_MIGRATE_FILE_FAILED,

                                 "Migrate file failed: %s: File has locks."

                                 " Skipping file migration", loc->path);

+                        *fop_errno = ENOTSUP;

                         ret = -1;

                         goto out;

                 }

@@ -561,17 +588,17 @@ __is_file_migratable (xlator_t *this, loc_t *loc,

         /* Check if file has hardlink*/

         ret = __check_file_has_hardlink (this, loc, stbuf, xattrs,

-                                         flags, defrag);

+                                         flags, defrag, fop_errno);

 out:

         return ret;

 }

 static int

-__dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struct iatt *stbuf,

-                                 fd_t **dst_fd, dict_t *xattr)

+__dht_rebalance_create_dst_file (xlator_t *this, xlator_t *to, xlator_t *from,

+                                 loc_t *loc, struct iatt *stbuf, fd_t **dst_fd,

+                                 dict_t *xattr, int *fop_errno)

 {

-        xlator_t    *this = NULL;

         int          ret  = -1;

         fd_t        *fd   = NULL;

         struct iatt  new_stbuf = {0,};

@@ -579,15 +606,21 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc

         dht_conf_t  *conf = NULL;

         dict_t      *dict = NULL;

-        this = THIS;

         conf = this->private;

         dict = dict_new ();

-        if (!dict)

+        if (!dict) {

+                *fop_errno = ENOMEM;

+                ret = -1;

+                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,

+                        DHT_MSG_NO_MEMORY, "dictionary allocation failed for"

+                        "path:%s", loc->path);

                 goto out;

-

+        }

         ret = dict_set_static_bin (dict, "gfid-req", stbuf->ia_gfid, 16);

         if (ret) {

+                *fop_errno = ENOMEM;

+                ret = -1;

                 gf_msg (this->name, GF_LOG_ERROR, 0,

                         DHT_MSG_DICT_SET_FAILED,

                         "%s: failed to set dictionary value: key = gfid-req",

@@ -597,6 +630,8 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc

         ret = dict_set_str (dict, conf->link_xattr_name, from->name);

         if (ret) {

+                *fop_errno = ENOMEM;

+                ret = -1;

                 gf_msg (this->name, GF_LOG_ERROR, 0,

                         DHT_MSG_DICT_SET_FAILED,

                         "%s: failed to set dictionary value: key = %s ",

@@ -606,11 +641,12 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc

         fd = fd_create (loc->inode, DHT_REBALANCE_PID);

         if (!fd) {

-                gf_msg (this->name, GF_LOG_ERROR, 0,

-                        DHT_MSG_MIGRATE_FILE_FAILED,

-                        "%s: fd create failed (destination) (%s)",

-                        loc->path, strerror (errno));

+                *fop_errno = ENOMEM;

                 ret = -1;

+                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,

+                        DHT_MSG_MIGRATE_FILE_FAILED,

+                        "%s: fd create failed (destination)",

+                        loc->path);

                 goto out;

         }

@@ -622,6 +658,7 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc

                                 DHT_MSG_GFID_MISMATCH,

                                 "file %s exists in %s with different gfid",

                                 loc->path, to->name);

+                        *fop_errno = EINVAL;

                         ret = -1;

                         goto out;

                 }

@@ -632,6 +669,7 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "%s: failed to lookup file (%s)",

                         loc->path, strerror (-ret));

+                *fop_errno = -ret;

                 ret = -1;

                 goto out;

         }

@@ -648,6 +686,7 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc

                                 DHT_MSG_MIGRATE_FILE_FAILED,

                                 "failed to open %s on %s",

                                 loc->path, to->name);

+                        *fop_errno = -ret;

                         ret = -1;

                         goto out;

                  }

@@ -659,6 +698,7 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc

                                 DHT_MSG_MIGRATE_FILE_FAILED,

                                 "failed to create %s on %s",

                                 loc->path, to->name);

+                        *fop_errno = -ret;

                         ret = -1;

                         goto out;

                 }

@@ -686,6 +726,7 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc

                                 "file %s exists in %s with different gfid,"

                                 "found in lookup after create",

                                 loc->path, to->name);

+                        *fop_errno = EINVAL;

                         ret = -1;

                         goto out;

                 }

@@ -696,23 +737,27 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc

                 gf_msg (this->name, GF_LOG_ERROR, 0,

                         DHT_MSG_MIGRATE_FILE_FAILED, "%s: file does not exists"

                         "on %s (%s)", loc->path, to->name, strerror (-ret));

+                *fop_errno = -ret;

                 ret = -1;

                 goto out;

         }

         ret = syncop_fsetxattr (to, fd, xattr, 0, NULL, NULL);

-        if (ret < 0)

+        if (ret < 0) {

+                *fop_errno = -ret;

                 gf_msg (this->name, GF_LOG_WARNING, 0,

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "%s: failed to set xattr on %s (%s)",

                         loc->path, to->name, strerror (-ret));

+        }

         /* TODO: Need to add a detailed comment about why we moved away from

         ftruncate.

         ret = syncop_ftruncate (to, fd, stbuf->ia_size, NULL, NULL);

-        if (ret < 0)

+        if (ret < 0) {

+                *fop_errno = -ret;

                 gf_msg (this->name, GF_LOG_ERROR, 0,

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "ftruncate failed for %s on %s (%s)",

@@ -722,11 +767,13 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc

         ret = syncop_fsetattr (to, fd, stbuf,

                                (GF_SET_ATTR_UID | GF_SET_ATTR_GID),

                                 NULL, NULL, NULL, NULL);

-        if (ret < 0)

+        if (ret < 0) {

+                *fop_errno = -ret;

                 gf_msg (this->name, GF_LOG_ERROR, 0,

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "chown failed for %s on %s (%s)",

                         loc->path, to->name, strerror (-ret));

+        }

         /* Fallocate does not work for size 0, hence the check. Anyway we don't

          * need to care about min-free-disk for 0 byte size file */

@@ -762,15 +809,14 @@ out:

 }

 static int

-__dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,

+__dht_check_free_space (xlator_t *this, xlator_t *to, xlator_t *from, loc_t *loc,

                         struct iatt *stbuf, int flag, dht_conf_t *conf,

                         gf_boolean_t *target_changed, xlator_t **new_subvol,

-                        gf_boolean_t *ignore_failure)

+                        gf_boolean_t *ignore_failure,  int *fop_errno)

 {

         struct statvfs  src_statfs = {0,};

         struct statvfs  dst_statfs = {0,};

         int             ret        = -1;

-        xlator_t       *this       = NULL;

         dict_t         *xdata      = NULL;

         dht_layout_t   *layout     = NULL;

         uint64_t        src_statfs_blocks = 1;

@@ -779,11 +825,10 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,

         double   post_percent = 0;

         int             i = 0;

-        this = THIS;

-

         xdata = dict_new ();

         if (!xdata) {

-                errno = ENOMEM;

+                *fop_errno = ENOMEM;

+                ret = -1;

                 gf_msg (this->name, GF_LOG_ERROR, ENOMEM,

                         DHT_MSG_NO_MEMORY,

                         "failed to allocate dictionary");

@@ -796,6 +841,7 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,

                         "Failed to set "

                         GF_INTERNAL_IGNORE_DEEM_STATFS" in dict");

                 ret = -1;

+                *fop_errno = ENOMEM;

                 goto out;

         }

@@ -805,6 +851,7 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "failed to get statfs of %s on %s (%s)",

                         loc->path, from->name, strerror (-ret));

+                *fop_errno = -ret;

                 ret = -1;

                 goto out;

         }

@@ -815,6 +862,7 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "failed to get statfs of %s on %s (%s)",

                         loc->path, to->name, strerror (-ret));

+                *fop_errno = -ret;

                 ret = -1;

                 goto out;

         }

@@ -857,6 +905,7 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,

                         /* this is not a 'failure', but we don't want to

                            consider this as 'success' too :-/ */

+                        *fop_errno = ENOSPC;

                         ret = -1;

                         goto out;

                 }

@@ -898,6 +947,7 @@ find_new_subvol:

         layout = dht_layout_get (this, loc->parent);

         if (!layout) {

                 gf_log (this->name, GF_LOG_ERROR, "Layout is NULL");

+                *fop_errno = EINVAL;

                 ret = -1;

                 goto out;

         }

@@ -925,6 +975,7 @@ find_new_subvol:

                 }

                 *target_changed = _gf_false;

+                *fop_errno = ENOSPC;

                 ret = -1;

                 goto out;

         } else {

@@ -944,7 +995,7 @@ out:

 static int

 __dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,

-                              uint64_t ia_size, int hole_exists)

+                              uint64_t ia_size, int hole_exists, int *fop_errno)

 {

         int            ret    = 0;

         int            count  = 0;

@@ -963,18 +1014,26 @@ __dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst

                                     offset, 0, &vector, &count, &iobref, NULL,

                                     NULL);

                 if (!ret || (ret < 0)) {

+                        *fop_errno = -ret;

                         break;

                 }

-                if (hole_exists)

+                if (hole_exists) {

                         ret = dht_write_with_holes (to, dst, vector, count,

-                                                    ret, offset, iobref);

-                else

+                                                    ret, offset, iobref,

+                                                    fop_errno);

+                } else {

                         ret = syncop_writev (to, dst, vector, count,

                                              offset, iobref, 0, NULL, NULL);

+                        if (ret < 0) {

+                                *fop_errno = -ret;

+                        }

+                }

+

                 if (ret < 0) {

                         break;

                 }

+

                 offset += ret;

                 total += ret;

@@ -998,7 +1057,7 @@ __dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst

 static int

 __tier_migrate_data (gf_defrag_info_t *defrag, xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,

-                     uint64_t ia_size, int hole_exists)

+                     uint64_t ia_size, int hole_exists, int *fop_errno)

 {

         int            ret    = 0;

         int            count  = 0;

@@ -1018,15 +1077,20 @@ __tier_migrate_data (gf_defrag_info_t *defrag, xlator_t *from, xlator_t *to, fd_

                                     offset, 0, &vector, &count, &iobref, NULL,

                                     NULL);

                 if (!ret || (ret < 0)) {

+                        *fop_errno = -ret;

                         break;

                 }

-                if (hole_exists)

+                if (hole_exists) {

                         ret = dht_write_with_holes (to, dst, vector, count,

-                                                    ret, offset, iobref);

-                else

+                                                    ret, offset, iobref,

+                                                    fop_errno);

+                } else {

                         ret = syncop_writev (to, dst, vector, count,

                                              offset, iobref, 0, NULL, NULL);

+                        *fop_errno = -ret;

+                }

+

                 if (gf_defrag_get_pause_state (&defrag->tier_conf) != TIER_RUNNING) {

                         gf_msg ("tier", GF_LOG_INFO, 0,

                                 DHT_MSG_TIER_PAUSED,

@@ -1060,18 +1124,17 @@ __tier_migrate_data (gf_defrag_info_t *defrag, xlator_t *from, xlator_t *to, fd_

 static int

-__dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,

+__dht_rebalance_open_src_file (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,

                                struct iatt *stbuf, fd_t **src_fd,

-                               gf_boolean_t *clean_src)

+                               gf_boolean_t *clean_src, int *fop_errno)

 {

+

         int          ret  = 0;

         fd_t        *fd   = NULL;

         dict_t      *dict = NULL;

-        xlator_t    *this = NULL;

         struct iatt  iatt = {0,};

         dht_conf_t  *conf = NULL;

-        this = THIS;

         conf = this->private;

         *clean_src = _gf_false;

@@ -1081,6 +1144,7 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,

                 gf_msg (this->name, GF_LOG_ERROR, 0,

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "%s: fd create failed (source)", loc->path);

+                *fop_errno = ENOMEM;

                 ret = -1;

                 goto out;

         }

@@ -1091,6 +1155,7 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,

                         DHT_MSG_MIGRATE_FILE_FAILED,

                         "failed to open file %s on %s (%s)",

                         loc->path, from->name, strerror (-ret));

+                *fop_errno = -ret;

                 ret = -1;

                 goto out;

         }

@@ -1102,14 +1167,22 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,

         ret = -1;

         dict = dict_new ();

-        if (!dict)

+        if (!dict) {

+                gf_msg (this->name, GF_LOG_ERROR, 0,

+                        DHT_MSG_MIGRATE_FILE_FAILED,

+                        "%s: Could not allocate memory for dict", loc->path);

+                *fop_errno = ENOMEM;

+                ret = -1;

                 goto out;

+        }

         ret = dict_set_str (dict, conf->link_xattr_name, to->name);

         if (ret) {

                 gf_log (this->name, GF_LOG_ERROR,

                         "failed to set xattr in dict for %s (linkto:%s)",

                         loc->path, to->name);

+                *fop_errno = ENOMEM;