From 2b6e6c234dffa72c9f2af747908b1e1f29080698 Mon Sep 17 00:00:00 2001
From: Ravishankar N <ravishankar@redhat.com>
Date: Thu, 25 Mar 2021 11:52:13 +0530
Subject: [PATCH 559/584] afr: make fsync post-op aware of inodelk count
 (#2273)

Problem:
Since commit bd540db1e, eager-locking was enabled for fsync. But on
certain VM workloads wit sharding enabled, shard xlator keeps sending
fsync on the base shard. This can cause blocked inodelks from other
clients (including shd) to time out due to call bail.

Fix:
Make afr fsync aware of inodelk count and not delay post-op + unlock
when inodelk count > 1, just like writev.

Code is restructured so that any fd based AFR_DATA_TRANSACTION can be made
aware by setting GLUSTERFS_INODELK_DOM_COUNT in xdata request.

Note: We do not know yet why VMs go in to paused state because of the
blocked inodelks but this patch should be a first step in reducing the
occurence.

Upstream patch details:
> https://github.com/gluster/glusterfs/pull/2273/
> Updates: #2198
> Change-Id: Ib91ebdd3101d590c326e69c829cf9335003e260b
> Signed-off-by: Ravishankar N <ravishankar@redhat.com>

BUG: 1943467
Change-Id: Id407ca54007e3bbb206a1d9431ebaf89a2167f74
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/c/rhs-glusterfs/+/244516
Tested-by: RHGS Build Bot <nigelb@redhat.com>
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
---
 xlators/cluster/afr/src/afr-inode-write.c | 40 ++++++++++++++++++-------------
 xlators/features/locks/src/posix.c        |  1 +
 2 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index df82b6e..962a7b1 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -42,6 +42,7 @@ __afr_inode_write_finalize(call_frame_t *frame, xlator_t *this)
     struct iatt *stbuf = NULL;
     afr_local_t *local = NULL;
     afr_private_t *priv = NULL;
+    afr_lock_t *lock = NULL;
     afr_read_subvol_args_t args = {
         0,
     };
@@ -50,6 +51,12 @@ __afr_inode_write_finalize(call_frame_t *frame, xlator_t *this)
     priv = this->private;
     GF_VALIDATE_OR_GOTO(this->name, local->inode, out);
 
+    if (local->update_num_inodelks &&
+        local->transaction.type == AFR_DATA_TRANSACTION) {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        lock->num_inodelks = local->num_inodelks;
+    }
+
     /*This code needs to stay till DHT sends fops on linked
      * inodes*/
     if (!inode_is_linked(local->inode)) {
@@ -134,6 +141,7 @@ __afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
 {
     afr_local_t *local = NULL;
     afr_private_t *priv = NULL;
+    int num_inodelks = 0;
 
     local = frame->local;
     priv = this->private;
@@ -146,8 +154,16 @@ __afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
 
     local->replies[child_index].op_ret = op_ret;
     local->replies[child_index].op_errno = op_errno;
-    if (xdata)
+    if (xdata) {
         local->replies[child_index].xdata = dict_ref(xdata);
+        if (dict_get_int32_sizen(xdata, GLUSTERFS_INODELK_COUNT,
+                                 &num_inodelks) == 0) {
+            if (num_inodelks > local->num_inodelks) {
+                local->num_inodelks = num_inodelks;
+                local->update_num_inodelks = _gf_true;
+            }
+        }
+    }
 
     if (op_ret >= 0) {
         if (prebuf)
@@ -284,7 +300,6 @@ afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
     afr_local_t *local = frame->local;
     uint32_t open_fd_count = 0;
     uint32_t write_is_append = 0;
-    int32_t num_inodelks = 0;
 
     LOCK(&frame->lock);
     {
@@ -306,15 +321,6 @@ afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
             local->open_fd_count = open_fd_count;
             local->update_open_fd_count = _gf_true;
         }
-
-        ret = dict_get_int32_sizen(xdata, GLUSTERFS_INODELK_COUNT,
-                                   &num_inodelks);
-        if (ret < 0)
-            goto unlock;
-        if (num_inodelks > local->num_inodelks) {
-            local->num_inodelks = num_inodelks;
-            local->update_num_inodelks = _gf_true;
-        }
     }
 unlock:
     UNLOCK(&frame->lock);
@@ -324,7 +330,6 @@ void
 afr_process_post_writev(call_frame_t *frame, xlator_t *this)
 {
     afr_local_t *local = NULL;
-    afr_lock_t *lock = NULL;
 
     local = frame->local;
 
@@ -343,11 +348,6 @@ afr_process_post_writev(call_frame_t *frame, xlator_t *this)
 
     if (local->update_open_fd_count)
         local->inode_ctx->open_fd_count = local->open_fd_count;
-    if (local->update_num_inodelks &&
-        local->transaction.type == AFR_DATA_TRANSACTION) {
-        lock = &local->inode_ctx->lock[local->transaction.type];
-        lock->num_inodelks = local->num_inodelks;
-    }
 }
 
 int
@@ -2516,6 +2516,12 @@ afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
     if (!local->xdata_req)
         goto out;
 
+    if (dict_set_str_sizen(local->xdata_req, GLUSTERFS_INODELK_DOM_COUNT,
+                           this->name)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
     local->fd = fd_ref(fd);
     ret = afr_set_inode_local(this, local, fd->inode);
     if (ret)
diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c
index cdd1ff7..22ef5b8 100644
--- a/xlators/features/locks/src/posix.c
+++ b/xlators/features/locks/src/posix.c
@@ -4943,6 +4943,7 @@ struct xlator_fops fops = {
     .rchecksum = pl_rchecksum,
     .statfs = pl_statfs,
     .fsyncdir = pl_fsyncdir,
+    .fsync = pl_fsync,
     .readdir = pl_readdir,
     .symlink = pl_symlink,
     .link = pl_link,
-- 
1.8.3.1