From 6fd12d2b182da4696d536f5b01a4b6701654b95f Mon Sep 17 00:00:00 2001

From: N Balachandran <nbalacha@redhat.com>

Date: Mon, 26 Jun 2017 21:12:56 +0530

Subject: [PATCH 559/566] cluster/dht: Check if fd is opened on dst subvol

If an fd is opened on a file, the file is migrated

and the cached subvol is updated in the inode_ctx

before an fd based fop is sent, the fop is sent to

the dst subvol on which the fd is not opened.

This causes the FOP to fail with EBADF.

Now, every fd based fop will check to see that the fd

has been opened on the dst subvol before winding it down.

> BUG: 1465075

> Signed-off-by: N Balachandran <nbalacha@redhat.com>

> Reviewed-on: https://review.gluster.org/17630

> Smoke: Gluster Build System <jenkins@build.gluster.org>

> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

> Reviewed-by: Susant Palai <spalai@redhat.com>

> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

Signed-off-by: N Balachandran <nbalacha@redhat.com>

Change-Id: I38ad5a7e0ecf86f74cf695e7e65942013ca2d86d

BUG: 1463907

Reviewed-on: https://code.engineering.redhat.com/gerrit/111154

Tested-by: Nithya Balachandran <nbalacha@redhat.com>

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

---

 xlators/cluster/dht/src/dht-common.c      |   8 +

 xlators/cluster/dht/src/dht-common.h      |  61 +++++++

 xlators/cluster/dht/src/dht-helper.c      | 280 ++++++++++++++++++++++++++++++

 xlators/cluster/dht/src/dht-inode-read.c  |  83 +++++++--

 xlators/cluster/dht/src/dht-inode-write.c | 130 +++++++++++---

 xlators/cluster/dht/src/dht-layout.c      |   5 +

 6 files changed, 537 insertions(+), 30 deletions(-)

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c

index 567990b..aebea2f 100644

--- a/xlators/cluster/dht/src/dht-common.c

+++ b/xlators/cluster/dht/src/dht-common.c

@@ -6774,6 +6774,14 @@ dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

                                            postparent, 1);

         }

+        ret = dht_fd_ctx_set (this, fd, prev);

+        if (ret != 0) {

+                gf_msg_debug (this->name, 0, "Possible fd leak. "

+                              "Could not set fd ctx for subvol %s",

+                              prev->name);

+        }

+

+

         ret = dht_layout_preset (this, prev, inode);

         if (ret != 0) {

                 gf_msg_debug (this->name, 0,

diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h

index 28e9bbf..8112b01 100644

--- a/xlators/cluster/dht/src/dht-common.h

+++ b/xlators/cluster/dht/src/dht-common.h

@@ -1274,4 +1274,65 @@ dht_remove_stale_linkto (void *data);

 int

 dht_remove_stale_linkto_cbk (int ret, call_frame_t *sync_frame, void *data);

+

+int

+dht_fd_ctx_set (xlator_t *this, fd_t *fd, xlator_t *subvol);

+

+int

+dht_check_and_open_fd_on_subvol (xlator_t *this, call_frame_t *frame);

+

+

+/* FD fop callbacks */

+

+int

+dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

+                int op_ret, int op_errno, struct iatt *prebuf,

+                struct iatt *postbuf, dict_t *xdata);

+

+int

+dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

+               int op_ret, int op_errno, dict_t *xdata);

+

+int

+dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

+                      int op_ret, int op_errno, struct iatt *prebuf,

+                      struct iatt *postbuf, dict_t *xdata);

+

+int

+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,

+                int op_ret, int op_errno, struct iatt *prebuf,

+                struct iatt *postbuf, dict_t *xdata);

+

+int

+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,

+                int op_ret, int op_errno, struct iatt *prebuf,

+                struct iatt *postbuf, dict_t *xdata);

+

+int

+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,

+                  int op_ret, int op_errno, struct iatt *prebuf,

+                  struct iatt *postbuf, dict_t *xdata);

+

+int

+dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

+                  int op_ret, int op_errno, struct iatt *prebuf,

+                  struct iatt *postbuf, dict_t *xdata);

+

+int

+dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,

+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,

+               dict_t *xdata);

+

+

+int

+dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

+               int op_ret, int op_errno,

+               struct iovec *vector, int count, struct iatt *stbuf,

+               struct iobref *iobref, dict_t *xdata);

+

+int

+dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

+                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata);

+

+

 #endif/* _DHT_H */

diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c

index f6cbe2c..50cdb83 100644

--- a/xlators/cluster/dht/src/dht-helper.c

+++ b/xlators/cluster/dht/src/dht-helper.c

@@ -267,6 +267,286 @@ dht_mig_info_is_invalid (xlator_t *current, xlator_t *src_subvol,

         return _gf_false;

 }

+

+

+/* Used to check if fd fops have the fd opened on the cached subvol

+ * This is required when:

+ * 1. an fd is opened on FILE1 on subvol1

+ * 2. the file is migrated to subvol2

+ * 3. a lookup updates the cached subvol in the inode_ctx to subvol2

+ * 4. a write comes on the fd

+ * The write is sent to subvol2 on an fd which has been opened only on fd1

+ * Since the migration phase checks don't kick in, the fop fails with EBADF

+ *

+ */

+

+

+int

+dht_check_and_open_fd_on_subvol_complete (int ret, call_frame_t *frame,

+                                          void *data)

+{

+        glusterfs_fop_t     fop     = 0;

+        dht_local_t        *local   = NULL;

+        xlator_t           *subvol  = NULL;

+        fd_t               *fd      = NULL;

+        int                 op_errno = -1;

+

+        local = frame->local;

+        fop = local->fop;

+        subvol = local->cached_subvol;

+        fd = local->fd;

+

+        if (ret) {

+                op_errno = local->op_errno;

+                goto handle_err;

+        }

+

+        switch (fop) {

+

+        case GF_FOP_WRITE:

+

+                STACK_WIND_COOKIE (frame, dht_writev_cbk, subvol, subvol,

+                                   subvol->fops->writev, fd,

+                                   local->rebalance.vector,

+                                   local->rebalance.count,

+                                   local->rebalance.offset,

+                                   local->rebalance.flags,

+                                   local->rebalance.iobref, local->xattr_req);

+                break;

+

+        case GF_FOP_FLUSH:

+

+                STACK_WIND (frame, dht_flush_cbk, subvol,

+                            subvol->fops->flush, fd, local->xattr_req);

+                break;

+

+        case GF_FOP_FSETATTR:

+

+                STACK_WIND_COOKIE (frame, dht_file_setattr_cbk, subvol,

+                                   subvol, subvol->fops->fsetattr, fd,

+                                   &local->rebalance.stbuf,

+                                   local->rebalance.flags,

+                                   local->xattr_req);

+                break;

+

+        case GF_FOP_ZEROFILL:

+                STACK_WIND_COOKIE (frame, dht_zerofill_cbk, subvol, subvol,

+                                   subvol->fops->zerofill, fd,

+                                   local->rebalance.offset,

+                                   local->rebalance.size, local->xattr_req);

+

+                break;

+

+        case GF_FOP_DISCARD:

+                STACK_WIND_COOKIE (frame, dht_discard_cbk, subvol, subvol,

+                                   subvol->fops->discard, local->fd,

+                                   local->rebalance.offset,

+                                   local->rebalance.size,

+                                   local->xattr_req);

+                break;

+

+        case GF_FOP_FALLOCATE:

+                STACK_WIND_COOKIE (frame, dht_fallocate_cbk, subvol, subvol,

+                                   subvol->fops->fallocate, fd,

+                                   local->rebalance.flags,

+                                   local->rebalance.offset,

+                                   local->rebalance.size,

+                                   local->xattr_req);

+                break;

+

+        case GF_FOP_FTRUNCATE:

+                STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol,

+                                   subvol->fops->ftruncate, fd,

+                                   local->rebalance.offset, local->xattr_req);

+                break;

+

+        case GF_FOP_FSYNC:

+                STACK_WIND_COOKIE (frame, dht_fsync_cbk, subvol, subvol,

+                                   subvol->fops->fsync, local->fd,

+                                   local->rebalance.flags, local->xattr_req);

+                break;

+

+        case GF_FOP_READ:

+                STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,

+                            local->fd, local->rebalance.size,

+                            local->rebalance.offset,

+                            local->rebalance.flags, local->xattr_req);

+                break;

+

+        case GF_FOP_FSTAT:

+                STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol,

+                                   subvol, subvol->fops->fstat, fd,

+                                   local->xattr_req);

+                break;

+

+        default:

+                break;

+

+        }

+

+        goto out;

+

+        /* Could not open the fd on the dst. Unwind */

+

+handle_err:

+

+        switch (fop) {

+

+        case GF_FOP_WRITE:

+                DHT_STACK_UNWIND (writev, frame, -1,

+                                  op_errno, NULL, NULL, NULL);

+                break;

+

+        case GF_FOP_FLUSH:

+                DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);

+                break;

+

+        case GF_FOP_FSETATTR:

+                DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno,

+                                  NULL, NULL, NULL);

+                break;

+

+        case GF_FOP_ZEROFILL:

+                DHT_STACK_UNWIND (zerofill, frame, -1, op_errno,

+                                  NULL, NULL, NULL);

+                break;

+

+        case GF_FOP_DISCARD:

+                DHT_STACK_UNWIND (discard, frame, -1, op_errno,

+                                  NULL, NULL, NULL);

+                break;

+

+        case GF_FOP_FALLOCATE:

+                DHT_STACK_UNWIND (fallocate, frame, -1, op_errno,

+                                  NULL, NULL, NULL);

+                break;

+

+        case GF_FOP_FTRUNCATE:

+                DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno,

+                                  NULL, NULL, NULL);

+                break;

+

+        case GF_FOP_FSYNC:

+                DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);

+                break;

+

+        case GF_FOP_READ:

+                DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL,

+                                  0, NULL, NULL, NULL);

+                break;

+

+        case GF_FOP_FSTAT:

+                DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);

+                break;

+

+        default:

+                break;

+        }

+

+out:

+

+        return 0;

+

+}

+

+

+/* Check once again if the fd has been opened on the cached subvol.

+ * If not, open and update the fd_ctx.

+ */

+

+int

+dht_check_and_open_fd_on_subvol_task (void *data)

+{

+        loc_t          loc        = {0,};

+        int            ret        = -1;

+        call_frame_t  *frame      = NULL;

+        dht_local_t   *local      = NULL;

+        fd_t          *fd         = NULL;

+        xlator_t      *this       = NULL;

+        xlator_t      *subvol     = NULL;

+

+

+        frame = data;

+        local = frame->local;

+        this = THIS;

+        fd = local->fd;

+        subvol = local->cached_subvol;

+

+

+        if (fd_is_anonymous (fd) || dht_fd_open_on_dst (this, fd, subvol)) {

+                ret = 0;

+                goto out;

+        }

+

+        gf_msg_debug (this->name, 0,

+                      "Opening fd (%p, flags=0%o) on file %s @ %s",

+                      fd, fd->flags, uuid_utoa (fd->inode->gfid),

+                      subvol->name);

+

+

+        loc.inode = inode_ref (fd->inode);

+        gf_uuid_copy (loc.gfid, fd->inode->gfid);

+

+        /* Open this on the dst subvol */

+

+        SYNCTASK_SETID(0, 0);

+

+        ret = syncop_open (subvol, &loc,

+                           (fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),

+                           fd, NULL, NULL);

+

+        if (ret < 0) {

+

+                gf_msg (this->name, GF_LOG_ERROR, -ret,

+                        DHT_MSG_OPEN_FD_ON_DST_FAILED,

+                        "Failed to open the fd"

+                        " (%p, flags=0%o) on file %s @ %s",

+                        fd, fd->flags, uuid_utoa (fd->inode->gfid),

+                        subvol->name);

+

+                local->op_errno = -ret;

+                ret = -1;

+

+        } else {

+                dht_fd_ctx_set (this, fd, subvol);

+        }

+

+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);

+out:

+        loc_wipe (&loc;;

+

+        return ret;

+}

+

+

+int

+dht_check_and_open_fd_on_subvol (xlator_t *this, call_frame_t *frame)

+{

+        int ret            = -1;

+        dht_local_t *local = NULL;

+

+/*

+        if (dht_fd_open_on_dst (this, fd, subvol))

+                goto out;

+*/

+        local = frame->local;

+

+        ret = synctask_new (this->ctx->env,

+                            dht_check_and_open_fd_on_subvol_task,

+                            dht_check_and_open_fd_on_subvol_complete,

+                            frame, frame);

+

+        if (ret) {

+                gf_msg (this->name, GF_LOG_ERROR, 0, 0,

+                        "Failed to create synctask"

+                        " to check and open fd=%p", local->fd);

+        }

+

+        return ret;

+}

+

+

+

 int

 dht_frame_return (call_frame_t *frame)

 {

diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c

index c53662d..58a0430 100644

--- a/xlators/cluster/dht/src/dht-inode-read.c

+++ b/xlators/cluster/dht/src/dht-inode-read.c

@@ -46,6 +46,12 @@ dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

                 goto out;

         }

+        /* Update ctx if the fd has been opened on the target*/

+        if (!op_ret && (local->call_cnt == 1)) {

+                dht_fd_ctx_set (this, fd, prev);

+                goto out;

+        }

+

         if (!op_ret || (local->call_cnt != 1))

                 goto out;

@@ -371,6 +377,7 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)

         dht_layout_t *layout = NULL;

         int           i = 0;

         int           call_cnt = 0;

+        int           ret      = -1;

         VALIDATE_OR_GOTO (frame, err);

@@ -397,9 +404,18 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)

                 local->call_cnt = 1;

                 subvol = local->cached_subvol;

+                if (dht_fd_open_on_dst (this, fd, subvol)) {

-                STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol, subvol,

-                                   subvol->fops->fstat, fd, xdata);

+                        STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol,

+                                           subvol, subvol->fops->fstat, fd,

+                                           xdata);

+

+                } else {

+                        ret = dht_check_and_open_fd_on_subvol (this, frame);

+

+                        if (ret)

+                                goto err;

+                }

                 return 0;

         }

@@ -529,6 +545,7 @@ dht_readv (call_frame_t *frame, xlator_t *this,

         xlator_t     *subvol = NULL;

         int           op_errno = -1;

         dht_local_t  *local = NULL;

+        int           ret      = -1;

         VALIDATE_OR_GOTO (frame, err);

         VALIDATE_OR_GOTO (this, err);

@@ -547,6 +564,7 @@ dht_readv (call_frame_t *frame, xlator_t *this,

                 op_errno = EINVAL;

                 goto err;

         }

+

         if (xdata)

                 local->xattr_req = dict_ref (xdata);

@@ -555,9 +573,19 @@ dht_readv (call_frame_t *frame, xlator_t *this,

         local->rebalance.flags  = flags;

         local->call_cnt = 1;

-        STACK_WIND (frame, dht_readv_cbk,

-                    subvol, subvol->fops->readv,

-                    fd, size, off, flags, xdata);

+        if (dht_fd_open_on_dst (this, fd, subvol)) {

+

+                STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,

+                            local->fd, local->rebalance.size,

+                            local->rebalance.offset,

+                            local->rebalance.flags, local->xattr_req);

+

+        } else {

+                ret = dht_check_and_open_fd_on_subvol (this, frame);

+                if (ret)

+                        goto err;

+        }

+

         return 0;

@@ -776,6 +804,7 @@ dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)

         xlator_t     *subvol = NULL;

         int           op_errno = -1;

         dht_local_t  *local = NULL;

+        int           ret      = -1;

         VALIDATE_OR_GOTO (frame, err);

         VALIDATE_OR_GOTO (this, err);

@@ -794,14 +823,24 @@ dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)

                 op_errno = EINVAL;

                 goto err;

         }

+

         if (xdata)

                 local->xattr_req = dict_ref (xdata);

         local->call_cnt = 1;

-        STACK_WIND (frame, dht_flush_cbk,

-                    subvol, subvol->fops->flush, fd, xdata);

+        if (dht_fd_open_on_dst (this, fd, subvol)) {

+

+                STACK_WIND (frame, dht_flush_cbk,

+                            subvol, subvol->fops->flush, fd, local->xattr_req);

+                return 0;

+        } else {

+

+                ret = dht_check_and_open_fd_on_subvol (this, frame);

+                if (ret)

+                        goto err;

+        }

         return 0;

 err:

@@ -935,6 +974,7 @@ dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,

         xlator_t     *subvol = NULL;

         int           op_errno = -1;

         dht_local_t  *local = NULL;

+        int           ret      = -1;

         VALIDATE_OR_GOTO (frame, err);

         VALIDATE_OR_GOTO (this, err);

@@ -954,8 +994,19 @@ dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,

         subvol = local->cached_subvol;

-        STACK_WIND_COOKIE (frame, dht_fsync_cbk, subvol, subvol,

-                           subvol->fops->fsync, fd, datasync, xdata);

+

+        if (dht_fd_open_on_dst (this, fd, subvol)) {

+

+                STACK_WIND_COOKIE (frame, dht_fsync_cbk, subvol, subvol,

+                                   subvol->fops->fsync, local->fd,

+                                   local->rebalance.flags, local->xattr_req);

+

+        } else {

+                ret = dht_check_and_open_fd_on_subvol (this, frame);

+                if (ret)

+                        goto err;

+        }

+

         return 0;

@@ -1073,6 +1124,13 @@ dht_lk (call_frame_t *frame, xlator_t *this,

                 op_errno = EINVAL;

                 goto err;

         }

+

+/*

+        local->cached_subvol = lock_subvol;

+        ret = dht_check_and_open_fd_on_subvol (this, frame);

+        if (ret)

+                goto err;

+*/

         if (xdata)

                 local->xattr_req = dict_ref (xdata);

@@ -1399,7 +1457,12 @@ dht_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,

                 goto err;

         }

-

+/*

+        local->cached_subvol = lock_subvol;

+        ret = dht_check_and_open_fd_on_subvol (this, frame);

+        if (ret)

+                goto err;

+*/

         STACK_WIND (frame, dht_finodelk_cbk, lock_subvol,

                     lock_subvol->fops->finodelk,

                     volume, fd, cmd, lock, xdata);

diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c

index 825822a..f925d4b 100644

--- a/xlators/cluster/dht/src/dht-inode-write.c

+++ b/xlators/cluster/dht/src/dht-inode-write.c

@@ -159,9 +159,10 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,

             struct iovec *vector, int count, off_t off, uint32_t flags,

             struct iobref *iobref, dict_t *xdata)

 {

-        xlator_t     *subvol = NULL;

+        xlator_t     *subvol   = NULL;

         int           op_errno = -1;

-        dht_local_t  *local = NULL;

+        dht_local_t  *local    = NULL;

+        int           ret      = -1;

         VALIDATE_OR_GOTO (frame, err);

         VALIDATE_OR_GOTO (this, err);

@@ -181,6 +182,8 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,

                 op_errno = EINVAL;

                 goto err;

         }

+

+

         if (xdata)

                 local->xattr_req = dict_ref (xdata);

@@ -191,9 +194,22 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,

         local->rebalance.iobref = iobref_ref (iobref);

         local->call_cnt = 1;

-        STACK_WIND_COOKIE (frame, dht_writev_cbk, subvol, subvol,

-                           subvol->fops->writev, fd, vector, count, off, flags,

-                           iobref, xdata);

+        if (dht_fd_open_on_dst (this, fd, subvol)) {

+

+                STACK_WIND_COOKIE (frame, dht_writev_cbk, subvol, subvol,

+                                   subvol->fops->writev, fd,

+                                   local->rebalance.vector,

+                                   local->rebalance.count,

+                                   local->rebalance.offset,

+                                   local->rebalance.flags,

+                                   local->rebalance.iobref, local->xattr_req);

+                return 0;

+

+        } else {

+                ret = dht_check_and_open_fd_on_subvol (this, frame);

+                if (ret)

+                        goto err;

+        }

         return 0;

@@ -218,6 +234,7 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         xlator_t     *dst_subvol = NULL;

         inode_t      *inode      = NULL;

+

         GF_VALIDATE_OR_GOTO ("dht", frame, err);

         GF_VALIDATE_OR_GOTO ("dht", this, out);

         GF_VALIDATE_OR_GOTO ("dht", frame->local, out);

@@ -368,6 +385,7 @@ dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,

                 op_errno = EINVAL;

                 goto err;

         }

+

         if (xdata)

                 local->xattr_req = dict_ref (xdata);

@@ -390,6 +408,8 @@ dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,

         xlator_t     *subvol = NULL;

         int           op_errno = -1;

         dht_local_t  *local = NULL;

+        int           ret   = -1;

+

         VALIDATE_OR_GOTO (frame, err);

         VALIDATE_OR_GOTO (this, err);

@@ -410,11 +430,23 @@ dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,

                 op_errno = EINVAL;

                 goto err;

         }

+

         if (xdata)

                 local->xattr_req = dict_ref (xdata);

-        STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol,

-                           subvol->fops->ftruncate, fd, offset, xdata);

+        if (dht_fd_open_on_dst (this, fd, subvol)) {

+

+                STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol,

+                                   subvol->fops->ftruncate, fd,

+                                   local->rebalance.offset, local->xattr_req);

+                return 0;

+

+        } else {

+

+                ret = dht_check_and_open_fd_on_subvol (this, frame);

+                if (ret)

+                        goto err;

+        }

         return 0;

@@ -548,12 +580,13 @@ out:

 }

 int

-dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,

+dht_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,

 	      off_t offset, size_t len, dict_t *xdata)

 {

         xlator_t     *subvol = NULL;

         int           op_errno = -1;

         dht_local_t  *local = NULL;

+        int           ret      = -1;

         VALIDATE_OR_GOTO (frame, err);

         VALIDATE_OR_GOTO (this, err);

@@ -577,12 +610,26 @@ dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,

                 op_errno = EINVAL;

                 goto err;

         }