From 1c094d1dc70dbc5d08181ef64fb300c95f331aec Mon Sep 17 00:00:00 2001

From: Krutika Dhananjay <kdhananj@redhat.com>

Date: Wed, 6 Dec 2017 16:55:33 +0530

Subject: [PATCH 171/180] features/shard: Upon FSYNC from upper layers, wind

 fsync on all changed shards

> Upstream: https://review.gluster.org/19566

> BUG: 1468483

> Change-Id: Ib74354f57a18569762ad45a51f182822a2537421

Change-Id: I93797a60e3449d02413d171babfb8e4292e3f2f6

BUG: 1493085

Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/131737

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

Tested-by: RHGS Build Bot <nigelb@redhat.com>

Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

---

 tests/bugs/shard/bug-1468483.t               |  58 +++

 tests/bugs/shard/shard-inode-refcount-test.t |   2 +-

 xlators/features/shard/src/shard-messages.h  |   9 +-

 xlators/features/shard/src/shard.c           | 533 +++++++++++++++++++++++++--

 xlators/features/shard/src/shard.h           |   6 +

 5 files changed, 569 insertions(+), 39 deletions(-)

 create mode 100644 tests/bugs/shard/bug-1468483.t

diff --git a/tests/bugs/shard/bug-1468483.t b/tests/bugs/shard/bug-1468483.t

new file mode 100644

index 0000000..e462b8d

--- /dev/null

+++ b/tests/bugs/shard/bug-1468483.t

@@ -0,0 +1,58 @@

+#!/bin/bash

+

+. $(dirname $0)/../../include.rc

+. $(dirname $0)/../../volume.rc

+. $(dirname $0)/../../common-utils.rc

+

+cleanup

+

+TEST glusterd

+TEST pidof glusterd

+TEST $CLI volume create $V0 $H0:$B0/${V0}0

+TEST $CLI volume set $V0 performance.write-behind off

+TEST $CLI volume set $V0 features.shard on

+TEST $CLI volume set $V0 features.shard-block-size 16MB

+TEST $CLI volume start $V0

+TEST $CLI volume profile $V0 start

+

+TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0

+TEST dd if=/dev/zero conv=fsync of=$M0/foo bs=1M count=100

+

+#This should ensure /.shard is created on the bricks.

+TEST stat $B0/${V0}0/.shard

+

+gfid_foo=$(get_gfid_string $M0/foo)

+

+TEST stat $B0/${V0}0/.shard/$gfid_foo.1

+TEST stat $B0/${V0}0/.shard/$gfid_foo.2

+TEST stat $B0/${V0}0/.shard/$gfid_foo.3

+TEST stat $B0/${V0}0/.shard/$gfid_foo.4

+TEST stat $B0/${V0}0/.shard/$gfid_foo.5

+TEST stat $B0/${V0}0/.shard/$gfid_foo.6

+

+# For a file with 7 shards, there should be 7 fsyncs on the brick. Without this

+# fix, I was seeing only 1 fsync (on the base shard alone).

+

+EXPECT "7" echo `$CLI volume profile $V0 info incremental | grep -w FSYNC | awk '{print $8}'`

+

+useradd -M test_user 2>/dev/null

+

+TEST touch $M0/bar

+

+# Change ownership to non-root on bar.

+TEST chown test_user:test_user $M0/bar

+

+TEST $CLI volume profile $V0 stop

+TEST $CLI volume profile $V0 start

+

+# Write 100M of data on bar as non-root.

+TEST run_cmd_as_user test_user "dd if=/dev/zero conv=fsync of=$M0/bar bs=1M count=100"

+

+EXPECT "7" echo `$CLI volume profile $V0 info incremental | grep -w FSYNC | awk '{print $8}'`

+

+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0

+userdel test_user

+TEST $CLI volume stop $V0

+TEST $CLI volume delete $V0

+

+cleanup

diff --git a/tests/bugs/shard/shard-inode-refcount-test.t b/tests/bugs/shard/shard-inode-refcount-test.t

index 6358097..03e0cc9 100644

--- a/tests/bugs/shard/shard-inode-refcount-test.t

+++ b/tests/bugs/shard/shard-inode-refcount-test.t

@@ -14,7 +14,7 @@ TEST $CLI volume start $V0

 TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0

-TEST dd if=/dev/zero of=$M0/one-plus-five-shards bs=1M count=23

+TEST dd if=/dev/zero conv=fsync of=$M0/one-plus-five-shards bs=1M count=23

 ACTIVE_INODES_BEFORE=$(get_mount_active_size_value $V0)

 TEST rm -f $M0/one-plus-five-shards

diff --git a/xlators/features/shard/src/shard-messages.h b/xlators/features/shard/src/shard-messages.h

index 588cb68..8e61630 100644

--- a/xlators/features/shard/src/shard-messages.h

+++ b/xlators/features/shard/src/shard-messages.h

@@ -40,7 +40,7 @@

  */

 #define GLFS_COMP_BASE_SHARD      GLFS_MSGID_COMP_SHARD

-#define GLFS_NUM_MESSAGES         18

+#define GLFS_NUM_MESSAGES         19

 #define GLFS_MSGID_END          (GLFS_COMP_BASE_SHARD + GLFS_NUM_MESSAGES + 1)

 #define glfs_msg_start_x GLFS_COMP_BASE_SHARD, "Invalid: Start of messages"

@@ -180,5 +180,12 @@

 */

 #define SHARD_MSG_INVALID_FOP                        (GLFS_COMP_BASE_SHARD + 18)

+/*!

+ * @messageid 133019

+ * @diagnosis

+ * @recommendedaction

+*/

+#define SHARD_MSG_MEMALLOC_FAILED                    (GLFS_COMP_BASE_SHARD + 19)

+

 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"

 #endif /* !_SHARD_MESSAGES_H_ */

diff --git a/xlators/features/shard/src/shard.c b/xlators/features/shard/src/shard.c

index 49cf04a..a661345 100644

--- a/xlators/features/shard/src/shard.c

+++ b/xlators/features/shard/src/shard.c

@@ -76,6 +76,7 @@ __shard_inode_ctx_get (inode_t *inode, xlator_t *this, shard_inode_ctx_t **ctx)

                 return ret;

         INIT_LIST_HEAD (&ctx_p->ilist);

+        INIT_LIST_HEAD (&ctx_p->to_fsync_list);

         ret = __inode_ctx_set (inode, this, (uint64_t *)&ctx_p);

         if (ret < 0) {

@@ -205,6 +206,65 @@ shard_inode_ctx_set_refreshed_flag (inode_t *inode, xlator_t *this)

         return ret;

 }

+int

+__shard_inode_ctx_add_to_fsync_list (inode_t *base_inode, xlator_t *this,

+                                     inode_t *shard_inode)

+{

+        int                 ret           = -1;

+        shard_inode_ctx_t  *base_ictx     = NULL;

+        shard_inode_ctx_t  *shard_ictx    = NULL;

+

+        ret = __shard_inode_ctx_get (base_inode, this, &base_ictx);

+        if (ret)

+                return ret;

+

+        ret = __shard_inode_ctx_get (shard_inode, this, &shard_ictx);

+        if (ret)

+                return ret;

+

+        if (shard_ictx->fsync_needed) {

+                shard_ictx->fsync_needed++;

+                return 1;

+        }

+

+        list_add_tail (&shard_ictx->to_fsync_list, &base_ictx->to_fsync_list);

+        shard_ictx->inode = shard_inode;

+        shard_ictx->fsync_needed++;

+        base_ictx->fsync_count++;

+        shard_ictx->base_inode = base_inode;

+

+        return 0;

+}

+

+int

+shard_inode_ctx_add_to_fsync_list (inode_t *base_inode, xlator_t *this,

+                                   inode_t *shard_inode)

+{

+        int ret = -1;

+

+        /* This ref acts as a refkeepr on the base inode. We

+         * need to keep this inode alive as it holds the head

+         * of the to_fsync_list.

+         */

+        inode_ref (base_inode);

+

+        LOCK (&base_inode->lock);

+        LOCK (&shard_inode->lock);

+        {

+                ret = __shard_inode_ctx_add_to_fsync_list (base_inode, this,

+                                                           shard_inode);

+        }

+        UNLOCK (&shard_inode->lock);

+        UNLOCK (&base_inode->lock);

+

+        /* Unref the base inode corresponding to the ref above, if the shard is

+         * found to be already part of the fsync list.

+         */

+        if (ret != 0)

+                inode_unref (base_inode);

+        return ret;

+}

+

 gf_boolean_t

 __shard_inode_ctx_needs_lookup (inode_t *inode, xlator_t *this)

 {

@@ -301,6 +361,40 @@ shard_inode_ctx_get_block_size (inode_t *inode, xlator_t *this,

 }

 int

+__shard_inode_ctx_get_fsync_count (inode_t *inode, xlator_t *this,

+                                   int *fsync_count)

+{

+        int                 ret      = -1;

+        uint64_t            ctx_uint = 0;

+        shard_inode_ctx_t  *ctx      = NULL;

+

+        ret = __inode_ctx_get (inode, this, &ctx_uint);

+        if (ret < 0)

+                return ret;

+

+        ctx = (shard_inode_ctx_t *) ctx_uint;

+

+        *fsync_count = ctx->fsync_needed;

+

+        return 0;

+}

+

+int

+shard_inode_ctx_get_fsync_count (inode_t *inode, xlator_t *this,

+                                int *fsync_count)

+{

+        int ret = -1;

+

+        LOCK (&inode->lock);

+        {

+                ret = __shard_inode_ctx_get_fsync_count (inode, this,

+                                                         fsync_count);

+        }

+        UNLOCK (&inode->lock);

+

+        return ret;

+}

+int

 __shard_inode_ctx_get_all (inode_t *inode, xlator_t *this,

                            shard_inode_ctx_t *ctx_out)

 {

@@ -482,15 +576,19 @@ out:

         return ret;

 }

-void

+inode_t *

 __shard_update_shards_inode_list (inode_t *linked_inode, xlator_t *this,

                                   inode_t *base_inode, int block_num)

 {

-        char                block_bname[256] = {0,};

-        inode_t            *lru_inode        = NULL;

-        shard_priv_t       *priv             = NULL;

-        shard_inode_ctx_t  *ctx              = NULL;

-        shard_inode_ctx_t  *lru_inode_ctx    = NULL;

+        char                block_bname[256]     = {0,};

+        inode_t            *lru_inode            = NULL;

+        shard_priv_t       *priv                 = NULL;

+        shard_inode_ctx_t  *ctx                  = NULL;

+        shard_inode_ctx_t  *lru_inode_ctx        = NULL;

+        shard_inode_ctx_t  *lru_base_inode_ctx   = NULL;

+        inode_t            *fsync_inode          = NULL;

+        inode_t            *lru_base_inode       = NULL;

+        gf_boolean_t        do_fsync             = _gf_false;

         priv = this->private;

@@ -510,6 +608,7 @@ __shard_update_shards_inode_list (inode_t *linked_inode, xlator_t *this,

                         ctx->block_num = block_num;

                         list_add_tail (&ctx->ilist, &priv->ilist_head);

                         priv->inode_count++;

+                        ctx->base_inode = base_inode;

                 } else {

                 /*If on the other hand there is no available slot for this inode

                  * in the list, delete the lru inode from the head of the list,

@@ -519,30 +618,56 @@ __shard_update_shards_inode_list (inode_t *linked_inode, xlator_t *this,

                                                           shard_inode_ctx_t,

                                                           ilist);

                         GF_ASSERT (lru_inode_ctx->block_num > 0);

+                        lru_base_inode = lru_inode_ctx->base_inode;

                         list_del_init (&lru_inode_ctx->ilist);

                         lru_inode = inode_find (linked_inode->table,

                                                 lru_inode_ctx->stat.ia_gfid);

-                        shard_make_block_bname (lru_inode_ctx->block_num,

-                                                lru_inode_ctx->base_gfid,

-                                                block_bname,

-                                                sizeof (block_bname));

-                        inode_unlink (lru_inode, priv->dot_shard_inode,

-                                      block_bname);

-                        /* The following unref corresponds to the ref held by

-                         * inode_find() above.

+                        /* If the lru inode was part of the pending-fsync list,

+                         * the base inode needs to be unref'd, the lru inode

+                         * deleted from fsync list and fsync'd in a new frame,

+                         * and then unlinked in memory and forgotten.

                          */

-                        inode_unref (lru_inode);

+                        LOCK (&lru_base_inode->lock);

+                        LOCK (&lru_inode->lock);

+                        {

+                                if (!list_empty(&lru_inode_ctx->to_fsync_list)) {

+                                        list_del_init (&lru_inode_ctx->to_fsync_list);

+                                        lru_inode_ctx->fsync_needed = 0;

+                                        do_fsync = _gf_true;

+                                        __shard_inode_ctx_get (lru_base_inode, this, &lru_base_inode_ctx);

+                                        lru_base_inode_ctx->fsync_count--;

+                                }

+                        }

+                        UNLOCK (&lru_inode->lock);

+                        UNLOCK (&lru_base_inode->lock);

+

+                        if (!do_fsync) {

+                                shard_make_block_bname (lru_inode_ctx->block_num,

+                                                        lru_inode_ctx->base_gfid,

+                                                        block_bname,

+                                                        sizeof (block_bname));

                         /* The following unref corresponds to the ref held at

-                         * the time the shard was created or looked up

+                         * the time the shard was added to the lru list.

+                         */

+                                inode_unref (lru_inode);

+                                inode_unlink (lru_inode, priv->dot_shard_inode,

+                                              block_bname);

+                                inode_forget (lru_inode, 0);

+                        } else {

+                                fsync_inode = lru_inode;

+                                inode_unref (lru_base_inode);

+                        }

+                        /* The following unref corresponds to the ref

+                         * held by inode_find() above.

                          */

                         inode_unref (lru_inode);

-                        inode_forget (lru_inode, 0);

                         /* For as long as an inode is in lru list, we try to

                          * keep it alive by holding a ref on it.

                          */

                         inode_ref (linked_inode);

                         gf_uuid_copy (ctx->base_gfid, base_inode->gfid);

                         ctx->block_num = block_num;

+                        ctx->base_inode = base_inode;

                         list_add_tail (&ctx->ilist, &priv->ilist_head);

                 }

         } else {

@@ -551,6 +676,7 @@ __shard_update_shards_inode_list (inode_t *linked_inode, xlator_t *this,

          */

                 list_move_tail (&ctx->ilist, &priv->ilist_head);

         }

+        return fsync_inode;

 }

 int

@@ -617,6 +743,85 @@ shard_common_inode_write_success_unwind (glusterfs_fop_t fop,

 }

 int

+shard_evicted_inode_fsync_cbk (call_frame_t *frame, void *cookie,

+                               xlator_t *this, int32_t op_ret, int32_t op_errno,

+                               struct iatt *prebuf, struct iatt *postbuf,

+                               dict_t *xdata)

+{

+        char                  block_bname[256] = {0,};

+        fd_t                 *anon_fd          = cookie;

+        inode_t              *shard_inode      = NULL;

+        shard_inode_ctx_t    *ctx              = NULL;

+        shard_priv_t         *priv             = NULL;

+

+        priv = this->private;

+        shard_inode = anon_fd->inode;

+

+        if (op_ret < 0) {

+                gf_msg (this->name, GF_LOG_WARNING, op_errno,

+                        SHARD_MSG_MEMALLOC_FAILED, "fsync failed on shard");

+                goto out;

+        }

+

+        LOCK (&priv->lock);

+        LOCK(&shard_inode->lock);

+        {

+                __shard_inode_ctx_get (shard_inode, this, &ctx;;

+                if ((list_empty(&ctx->to_fsync_list)) &&

+                    (list_empty(&ctx->ilist))) {

+                        shard_make_block_bname (ctx->block_num,

+                                                shard_inode->gfid, block_bname,

+                                                sizeof (block_bname));

+                        inode_unlink (shard_inode, priv->dot_shard_inode,

+                                      block_bname);

+                        /* The following unref corresponds to the ref held by

+                         * inode_link() at the time the shard was created or

+                         * looked up

+                         */

+                        inode_unref (shard_inode);

+                        inode_forget (shard_inode, 0);

+                }

+        }

+        UNLOCK(&shard_inode->lock);

+        UNLOCK(&priv->lock);

+

+out:

+        if (anon_fd)

+                fd_unref (anon_fd);

+        STACK_DESTROY (frame->root);

+        return 0;

+}

+

+int

+shard_initiate_evicted_inode_fsync (xlator_t *this, inode_t *inode)

+{

+        fd_t             *anon_fd     = NULL;

+        call_frame_t     *fsync_frame = NULL;

+

+        fsync_frame = create_frame (this, this->ctx->pool);

+        if (!fsync_frame) {

+                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,

+                        SHARD_MSG_MEMALLOC_FAILED, "Failed to create new frame "

+                        "to fsync shard");

+                return -1;

+        }

+

+        anon_fd = fd_anonymous (inode);

+        if (!anon_fd) {

+                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,

+                        SHARD_MSG_MEMALLOC_FAILED, "Failed to create anon fd to"

+                        " fsync shard");

+                STACK_DESTROY (fsync_frame->root);

+                return -1;

+        }

+

+        STACK_WIND_COOKIE (fsync_frame, shard_evicted_inode_fsync_cbk, anon_fd,

+                           FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,

+                           anon_fd, 1, NULL);

+        return 0;

+}

+

+int

 shard_common_resolve_shards (call_frame_t *frame, xlator_t *this,

                              shard_post_resolve_fop_handler_t post_res_handler)

 {

@@ -625,6 +830,7 @@ shard_common_resolve_shards (call_frame_t *frame, xlator_t *this,

         char                  path[PATH_MAX] = {0,};

         inode_t              *inode          = NULL;

         inode_t              *res_inode      = NULL;

+        inode_t              *fsync_inode    = NULL;

         shard_priv_t         *priv           = NULL;

         shard_local_t        *local          = NULL;

@@ -661,20 +867,22 @@ shard_common_resolve_shards (call_frame_t *frame, xlator_t *this,

                          */

                         LOCK(&priv->lock);

                         {

-                                __shard_update_shards_inode_list (inode, this,

+                                fsync_inode = __shard_update_shards_inode_list (inode,

+                                                                  this,

                                                                   res_inode,

                                                                 shard_idx_iter);

                         }

                         UNLOCK(&priv->lock);

                         shard_idx_iter++;

-

+                        if (fsync_inode)

+                                shard_initiate_evicted_inode_fsync (this,

+                                                                    fsync_inode);

                          continue;

                 } else {

                         local->call_count++;

                         shard_idx_iter++;

                 }

         }

-

 out:

         post_res_handler (frame, this);

         return 0;

@@ -1657,6 +1865,7 @@ shard_link_block_inode (shard_local_t *local, int block_num, inode_t *inode,

         char            block_bname[256] = {0,};

         inode_t        *linked_inode     = NULL;

         xlator_t       *this             = NULL;

+        inode_t        *fsync_inode      = NULL;

         shard_priv_t   *priv             = NULL;

         this = THIS;

@@ -1674,10 +1883,14 @@ shard_link_block_inode (shard_local_t *local, int block_num, inode_t *inode,

         LOCK(&priv->lock);

         {

-                __shard_update_shards_inode_list (linked_inode, this,

-                                                  local->loc.inode, block_num);

+                fsync_inode = __shard_update_shards_inode_list (linked_inode,

+                                                                this,

+                                                                local->loc.inode,

+                                                                block_num);

         }

         UNLOCK(&priv->lock);

+        if (fsync_inode)

+                shard_initiate_evicted_inode_fsync (this, fsync_inode);

 }

 int

@@ -2120,6 +2333,7 @@ shard_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,

         local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();

         if (!local->xattr_req)

                 goto err;

+        local->resolver_base_inode = loc->inode;

         shard_lookup_base_file (frame, this, &local->loc,

                                 shard_post_lookup_truncate_handler);

@@ -2172,6 +2386,7 @@ shard_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,

         local->loc.inode = inode_ref (fd->inode);

         gf_uuid_copy (local->loc.gfid, fd->inode->gfid);

+        local->resolver_base_inode = fd->inode;

         shard_lookup_base_file (frame, this, &local->loc,

                                 shard_post_lookup_truncate_handler);

@@ -2509,32 +2724,48 @@ shard_unlink_block_inode (shard_local_t *local, int shard_block_num)

 {

         char                  block_bname[256]  = {0,};

         inode_t              *inode             = NULL;

+        inode_t              *base_inode        = NULL;

         xlator_t             *this              = NULL;

         shard_priv_t         *priv              = NULL;

         shard_inode_ctx_t    *ctx               = NULL;

+        shard_inode_ctx_t    *base_ictx         = NULL;

+        gf_boolean_t          unlink_unref_forget = _gf_false;

         this = THIS;

         priv = this->private;

         inode = local->inode_list[shard_block_num - local->first_block];

+        base_inode = local->resolver_base_inode;

         shard_make_block_bname (shard_block_num, (local->loc.inode)->gfid,

                                 block_bname, sizeof (block_bname));

         LOCK(&priv->lock);

+        LOCK(&base_inode->lock);

+        LOCK(&inode->lock);

         {

-                shard_inode_ctx_get (inode, this, &ctx;;

+                __shard_inode_ctx_get (inode, this, &ctx;;

                 if (!list_empty (&ctx->ilist)) {

                         list_del_init (&ctx->ilist);

                         priv->inode_count--;

                         GF_ASSERT (priv->inode_count >= 0);

-                        inode_unlink (inode, priv->dot_shard_inode, block_bname);

-                        inode_unref (inode);

-                        inode_forget (inode, 0);

+                        unlink_unref_forget = _gf_true;

+                }

+                if (ctx->fsync_needed) {

+                        inode_unref (base_inode);

+                        list_del_init (&ctx->to_fsync_list);

+                        __shard_inode_ctx_get (base_inode, this, &base_ictx);

+                        base_ictx->fsync_count--;

                 }

         }

+        UNLOCK(&inode->lock);

+        UNLOCK(&base_inode->lock);

+        if (unlink_unref_forget) {

+                inode_unlink (inode, priv->dot_shard_inode, block_bname);

+                inode_unref (inode);

+                inode_forget (inode, 0);

+        }

         UNLOCK(&priv->lock);

-

 }

 int

@@ -2752,6 +2983,7 @@ shard_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,

         local->xflag = xflag;

         local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();

         local->block_size = block_size;

+        local->resolver_base_inode = loc->inode;

         local->fop = GF_FOP_UNLINK;

         if (!this->itable)

                 this->itable = (local->loc.inode)->table;

@@ -2988,6 +3220,7 @@ shard_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,

         frame->local = local;

         loc_copy (&local->loc, oldloc);

         loc_copy (&local->loc2, newloc);

+        local->resolver_base_inode = newloc->inode;

         local->fop = GF_FOP_RENAME;

         local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new();

         if (!local->xattr_req)

@@ -3754,6 +3987,10 @@ shard_common_inode_write_do_cbk (call_frame_t *frame, void *cookie,

                         local->delta_size += (post->ia_size - pre->ia_size);

                         shard_inode_ctx_set (local->fd->inode, this, post, 0,

                                              SHARD_MASK_TIMES);

+                        if (local->fd->inode != anon_fd->inode)

+                                shard_inode_ctx_add_to_fsync_list (local->fd->inode,

+                                                                   this,

+                                                                   anon_fd->inode);

                 }

         }

         UNLOCK (&frame->lock);

@@ -4204,18 +4441,198 @@ shard_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)

 }

 int

-shard_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,

-                 struct iatt *postbuf, dict_t *xdata)

+__shard_get_timestamps_from_inode_ctx (shard_local_t *local, inode_t *inode,

+                                       xlator_t *this)

 {

-        if (op_ret < 0)

+        int                   ret      = -1;

+        uint64_t              ctx_uint = 0;

+        shard_inode_ctx_t    *ctx      = NULL;

+

+        ret = __inode_ctx_get (inode, this, &ctx_uint);

+        if (ret < 0)

+                return ret;

+

+        ctx = (shard_inode_ctx_t *) ctx_uint;

+

+        local->postbuf.ia_ctime = ctx->stat.ia_ctime;

+        local->postbuf.ia_ctime_nsec = ctx->stat.ia_ctime_nsec;

+        local->postbuf.ia_atime = ctx->stat.ia_atime;

+        local->postbuf.ia_atime_nsec = ctx->stat.ia_atime_nsec;

+        local->postbuf.ia_mtime = ctx->stat.ia_mtime;

+        local->postbuf.ia_mtime_nsec = ctx->stat.ia_mtime_nsec;

+

+        return 0;

+}

+

+int

+shard_get_timestamps_from_inode_ctx (shard_local_t *local, inode_t *inode,

+                                     xlator_t *this)

+{

+        int ret = 0;

+

+        LOCK (&inode->lock);

+        {

+                ret = __shard_get_timestamps_from_inode_ctx (local, inode,

+                                                             this);

+        }

+        UNLOCK (&inode->lock);

+

+        return ret;

+}

+

+int

+shard_fsync_shards_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,

+                       struct iatt *postbuf, dict_t *xdata)

+{

+        int                    call_count  = 0;

+        uint64_t               fsync_count = 0;

+        fd_t                  *anon_fd     = cookie;

+        shard_local_t         *local       = NULL;

+        shard_inode_ctx_t     *ctx         = NULL;

+        shard_inode_ctx_t     *base_ictx   = NULL;

+        inode_t               *base_inode  = NULL;

+

+        local = frame->local;

+        base_inode = local->fd->inode;

+

+        if (local->op_ret < 0)

                 goto out;

-        /* To-Do: Wind fsync on all shards of the file */

-        postbuf->ia_ctime = 0;

+        LOCK (&frame->lock);

+        {

+                if (op_ret < 0) {

+                        local->op_ret = op_ret;

+                        local->op_errno = op_errno;

+                        goto out;

+                }

+                shard_inode_ctx_set (local->fd->inode, this, postbuf, 0,

+                                     SHARD_MASK_TIMES);

+        }

+        UNLOCK (&frame->lock);

+        fd_ctx_get (anon_fd, this, &fsync_count);

 out:

-        SHARD_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,

-                            xdata);

+        if (base_inode != anon_fd->inode) {

+                LOCK (&base_inode->lock);

+                LOCK (&anon_fd->inode->lock);

+                {

+                        __shard_inode_ctx_get (anon_fd->inode, this, &ctx;;

+                        __shard_inode_ctx_get (base_inode, this, &base_ictx);

+                        if (op_ret == 0)

+                                ctx->fsync_needed -= fsync_count;

+                        GF_ASSERT (ctx->fsync_needed >= 0);

+                        list_del_init (&ctx->to_fsync_list);

+                        if (ctx->fsync_needed != 0) {

+                                list_add_tail (&ctx->to_fsync_list,

+                                               &base_ictx->to_fsync_list);

+                                base_ictx->fsync_count++;

+                        }

+                }

+                UNLOCK (&anon_fd->inode->lock);

+                UNLOCK (&base_inode->lock);

+        }

+        if (anon_fd)

+                fd_unref (anon_fd);

+

+        call_count = shard_call_count_return (frame);

+        if (call_count != 0)

+                return 0;

+

+        if (local->op_ret < 0) {

+                SHARD_STACK_UNWIND (fsync, frame, local->op_ret,

+                                    local->op_errno, NULL, NULL, NULL);

+        } else {

+                shard_get_timestamps_from_inode_ctx (local, base_inode, this);

+                SHARD_STACK_UNWIND (fsync, frame, local->op_ret,

+                                    local->op_errno, &local->prebuf,

+                                    &local->postbuf, local->xattr_rsp);