From 4a6bdc7f785b8a030142d8036a13ae2f78984934 Mon Sep 17 00:00:00 2001

From: Krutika Dhananjay <kdhananj@redhat.com>

Date: Thu, 6 Apr 2017 18:10:41 +0530

Subject: [PATCH 308/316] features/shard: Fix vm corruption upon fix-layout

        Backport of: https://review.gluster.org/17010

shard's writev implementation, as part of identifying

presence of participant shards that aren't in memory,

first sends an MKNOD on these shards, and upon EEXIST error,

looks up the shards before proceeding with the writes.

The VM corruption was caused when the following happened:

1. DHT had n subvolumes initially.

2. Upon add-brick + fix-layout, the layout of .shard changed

   although the existing shards under it were yet to be migrated

   to their new hashed subvolumes.

3. During this time, there were writes on the VM falling in regions

   of the file whose corresponding shards were already existing under

   .shard.

4. Sharding xl sent MKNOD on these shards, now creating them in their

   new hashed subvolumes although there already exist shard blocks for

   this region with valid data.

5. All subsequent writes were wound on these newly created copies.

The net outcome is that both copies of the shard didn't have the correct

data. This caused the affected VMs to be unbootable.

FIX:

For want of better alternatives in DHT, the fix changes shard fops to do

a LOOKUP before the MKNOD and upon EEXIST error, perform another lookup.

Change-Id: I4086e7a9c27c9325b3830f4274be87847283a9f2

BUG: 1439753

RCA'd-by: Raghavendra Gowdappa <rgowdapp@redhat.com>

Reported-by: Mahdi Adnan <mahdi.adnan@outlook.com>

Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/105195

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

---

 xlators/features/shard/src/shard.c | 154 +++++++++++++++++++++++--------------

 xlators/features/shard/src/shard.h |   1 +

 2 files changed, 96 insertions(+), 59 deletions(-)

diff --git a/xlators/features/shard/src/shard.c b/xlators/features/shard/src/shard.c

index 9504f12..eabed90 100644

--- a/xlators/features/shard/src/shard.c

+++ b/xlators/features/shard/src/shard.c

@@ -1693,11 +1693,30 @@ shard_common_lookup_shards_cbk (call_frame_t *frame, void *cookie,

         if (op_ret < 0) {

                 /* Ignore absence of shards in the backend in truncate fop. */

-                if (((local->fop == GF_FOP_TRUNCATE) ||

-                    (local->fop == GF_FOP_FTRUNCATE) ||

-                    (local->fop == GF_FOP_RENAME) ||

-                    (local->fop == GF_FOP_UNLINK)) && (op_errno == ENOENT))

-                        goto done;

+                switch (local->fop) {

+                case GF_FOP_TRUNCATE:

+                case GF_FOP_FTRUNCATE:

+                case GF_FOP_RENAME:

+                case GF_FOP_UNLINK:

+                        if (op_errno == ENOENT)

+                                goto done;

+                        break;

+                case GF_FOP_WRITE:

+                case GF_FOP_READ:

+                case GF_FOP_ZEROFILL:

+                case GF_FOP_DISCARD:

+                case GF_FOP_FALLOCATE:

+                        if ((!local->first_lookup_done) &&

+                            (op_errno == ENOENT)) {

+                                local->create_count++;

+                                goto done;

+                        }

+                        break;

+                default:

+                        break;

+                }

+

+                /* else */

                 gf_msg (this->name, GF_LOG_ERROR, op_errno,

                         SHARD_MSG_LOOKUP_SHARD_FAILED, "Lookup on shard %d "

                         "failed. Base file gfid = %s", shard_block_num,

@@ -1714,6 +1733,8 @@ shard_common_lookup_shards_cbk (call_frame_t *frame, void *cookie,

 done:

         call_count = shard_call_count_return (frame);

         if (call_count == 0) {

+                if (!local->first_lookup_done)

+                        local->first_lookup_done = _gf_true;

                 if (local->op_ret < 0)

                         goto unwind;

                 else

@@ -3197,47 +3218,6 @@ next:

 }

 int

-shard_post_lookup_shards_readv_handler (call_frame_t *frame, xlator_t *this)

-{

-        shard_local_t *local = NULL;

-

-        local = frame->local;

-

-        if (local->op_ret < 0) {

-                SHARD_STACK_UNWIND (readv, frame, local->op_ret,

-                                    local->op_errno, NULL, 0, NULL, NULL, NULL);

-                return 0;

-        }

-

-        shard_readv_do (frame, this);

-

-        return 0;

-}

-

-int

-shard_post_mknod_readv_handler (call_frame_t *frame, xlator_t *this)

-{

-        shard_local_t *local = NULL;

-

-        local = frame->local;

-

-        if (local->op_ret < 0) {

-                SHARD_STACK_UNWIND (readv, frame, local->op_ret,

-                                    local->op_errno, NULL, 0, NULL, NULL, NULL);

-                return 0;

-        }

-

-        if (!local->eexist_count) {

-                shard_readv_do (frame, this);

-        } else {

-                local->call_count = local->eexist_count;

-                shard_common_lookup_shards (frame, this, local->loc.inode,

-                                        shard_post_lookup_shards_readv_handler);

-        }

-        return 0;

-}

-

-int

 shard_common_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

                         int32_t op_ret, int32_t op_errno, inode_t *inode,

                         struct iatt *buf, struct iatt *preparent,

@@ -3267,6 +3247,7 @@ done:

         call_count = shard_call_count_return (frame);

         if (call_count == 0) {

                 SHARD_UNSET_ROOT_FS_ID (frame, local);

+                local->create_count = 0;

                 local->post_mknod_handler (frame, this);

         }

@@ -3397,6 +3378,55 @@ err:

 }

 int

+shard_post_mknod_readv_handler (call_frame_t *frame, xlator_t *this);

+

+int

+shard_post_lookup_shards_readv_handler (call_frame_t *frame, xlator_t *this)

+{

+        shard_local_t *local = NULL;

+

+        local = frame->local;

+

+        if (local->op_ret < 0) {

+                SHARD_STACK_UNWIND (readv, frame, local->op_ret,

+                                    local->op_errno, NULL, 0, NULL, NULL, NULL);

+                return 0;

+        }

+

+        if (local->create_count) {

+                shard_common_resume_mknod (frame, this,

+                                           shard_post_mknod_readv_handler);

+        } else {

+                shard_readv_do (frame, this);

+        }

+

+        return 0;

+}

+

+int

+shard_post_mknod_readv_handler (call_frame_t *frame, xlator_t *this)

+{

+        shard_local_t *local = NULL;

+

+        local = frame->local;

+

+        if (local->op_ret < 0) {

+                SHARD_STACK_UNWIND (readv, frame, local->op_ret,

+                                    local->op_errno, NULL, 0, NULL, NULL, NULL);

+                return 0;

+        }

+

+        if (!local->eexist_count) {

+                shard_readv_do (frame, this);

+        } else {

+                local->call_count = local->eexist_count;

+                shard_common_lookup_shards (frame, this, local->loc.inode,

+                                        shard_post_lookup_shards_readv_handler);

+        }

+        return 0;

+}

+

+int

 shard_post_resolve_readv_handler (call_frame_t *frame, xlator_t *this)

 {

         shard_local_t *local = NULL;

@@ -3422,9 +3452,9 @@ shard_post_resolve_readv_handler (call_frame_t *frame, xlator_t *this)

         }

         if (local->call_count) {

-                local->create_count = local->call_count;

-                shard_common_resume_mknod (frame, this,

-                                           shard_post_mknod_readv_handler);

+                shard_common_lookup_shards (frame, this,

+                                            local->resolver_base_inode,

+                                        shard_post_lookup_shards_readv_handler);

         } else {

                 shard_readv_do (frame, this);

         }

@@ -3575,14 +3605,11 @@ shard_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,

         shard_lookup_base_file (frame, this, &local->loc,

                                 shard_post_lookup_readv_handler);

-

         return 0;

-

 err:

         SHARD_STACK_UNWIND (readv, frame, -1, ENOMEM, NULL, 0, NULL, NULL,

                             NULL);

         return 0;

-

 }

 int

@@ -3875,6 +3902,10 @@ next:

 }

 int

+shard_common_inode_write_post_mknod_handler (call_frame_t *frame,

+                                             xlator_t *this);

+

+int

 shard_common_inode_write_post_lookup_shards_handler (call_frame_t *frame,

                                                      xlator_t *this)

 {

@@ -3889,7 +3920,12 @@ shard_common_inode_write_post_lookup_shards_handler (call_frame_t *frame,

                 return 0;

         }

-        shard_common_inode_write_do (frame, this);

+        if (local->create_count) {

+                shard_common_resume_mknod (frame, this,

+                                   shard_common_inode_write_post_mknod_handler);

+        } else {

+                shard_common_inode_write_do (frame, this);

+        }

         return 0;

 }

@@ -3937,11 +3973,13 @@ shard_common_inode_write_post_lookup_handler (call_frame_t *frame,

         local->postbuf = local->prebuf;

-        if (local->create_count)

-                shard_common_resume_mknod (frame, this,

-                                   shard_common_inode_write_post_mknod_handler);

-        else

+        if (local->call_count) {

+                shard_common_lookup_shards (frame, this,

+                                            local->resolver_base_inode,

+                           shard_common_inode_write_post_lookup_shards_handler);

+        } else {

                 shard_common_inode_write_do (frame, this);

+        }

         return 0;

 }

@@ -3961,8 +3999,6 @@ shard_common_inode_write_post_resolve_handler (call_frame_t *frame,

                 return 0;

         }

-        local->create_count = local->call_count;

-

         shard_lookup_base_file (frame, this, &local->loc,

                                 shard_common_inode_write_post_lookup_handler);

         return 0;

diff --git a/xlators/features/shard/src/shard.h b/xlators/features/shard/src/shard.h

index 09232a4..7319598 100644

--- a/xlators/features/shard/src/shard.h

+++ b/xlators/features/shard/src/shard.h

@@ -255,6 +255,7 @@ typedef struct shard_local {

                 shard_lock_t *shard_lock;

         } lock;

         inode_t *resolver_base_inode;

+        gf_boolean_t first_lookup_done;

 } shard_local_t;

 typedef struct shard_inode_ctx {

-- 

1.8.3.1