From 30993baa95ef0eac8abd461f12839247b76d6e4c Mon Sep 17 00:00:00 2001

From: Raghavendra G <rgowdapp@redhat.com>

Date: Mon, 23 Apr 2018 19:21:20 +0530

Subject: [PATCH 248/260] performance/md-cache: purge cache on ENOENT/ESTALE

 errors

If not, next lookup could be served from cache and can be success,

which is wrong. This can affect retry logic of VFS when it receives an

ESTALE.

> Change-Id: Iad8e564d666aa4172823343f19a60c11e4416ef6

> Signed-off-by: Raghavendra G <rgowdapp@redhat.com>

> Fixes: bz#1571593

(cherry picked from commit 6912441543192a366f264b61d97d50faf3815a28)

Change-Id: Iad8e564d666aa4172823343f19a60c11e4416ef6

BUG: 1546717

Signed-off-by: Raghavendra G <rgowdapp@redhat.com>

Fixes: bz#1546717

upstream: https://review.gluster.org/19926

Reviewed-on: https://code.engineering.redhat.com/gerrit/136806

Tested-by: RHGS Build Bot <nigelb@redhat.com>

Reviewed-by: Poornima Gurusiddaiah <pgurusid@redhat.com>

Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>

---

 xlators/performance/md-cache/src/md-cache.c | 624 ++++++++++++++++++++++++----

 1 file changed, 545 insertions(+), 79 deletions(-)

diff --git a/xlators/performance/md-cache/src/md-cache.c b/xlators/performance/md-cache/src/md-cache.c

index feab357..9d2eea6 100644

--- a/xlators/performance/md-cache/src/md-cache.c

+++ b/xlators/performance/md-cache/src/md-cache.c

@@ -24,11 +24,18 @@

 #include "md-cache-messages.h"

 #include "statedump.h"

 #include "atomic.h"

+#include "timespec.h"

 /* TODO:

    - cache symlink() link names and nuke symlink-cache

    - send proper postbuf in setattr_cbk even when op_ret = -1

 */

+struct mdc_statfs_cache {

+        pthread_mutex_t lock;

+        gf_boolean_t initialized;

+        struct timespec last_refreshed;

+        struct statvfs buf;

+};

 struct mdc_statistics {

         gf_atomic_t stat_hit; /* No. of times lookup/stat was served from

@@ -65,6 +72,8 @@ struct mdc_conf {

         time_t last_child_down;

         gf_lock_t lock;

         struct mdc_statistics mdc_counter;

+        gf_boolean_t cache_statfs;

+        struct mdc_statfs_cache statfs_cache;

 };

@@ -1062,20 +1071,165 @@ mdc_xattr_satisfied (xlator_t *this, dict_t *req, dict_t *rsp)

         return pair.ret;

 }

+static void

+mdc_cache_statfs (xlator_t *this, struct statvfs *buf)

+{

+        struct mdc_conf *conf = this->private;

+

+        pthread_mutex_lock (&conf->statfs_cache.lock);

+        {

+                memcpy (&conf->statfs_cache.buf, buf, sizeof (struct statvfs));

+                clock_gettime (CLOCK_MONOTONIC,

+                               &conf->statfs_cache.last_refreshed);

+                conf->statfs_cache.initialized = _gf_true;

+        }

+        pthread_mutex_unlock (&conf->statfs_cache.lock);

+}

 int

-mdc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

-                int32_t op_ret,	int32_t op_errno, inode_t *inode,

-                struct iatt *stbuf, dict_t *dict, struct iatt *postparent)

+mdc_load_statfs_info_from_cache (xlator_t *this, struct statvfs **buf)

 {

-        mdc_local_t *local = NULL;

         struct mdc_conf *conf = this->private;

+        struct timespec now;

+        double cache_age = 0.0;

+        int ret = 0;

+

+        if (!buf || !conf) {

+                ret = -1;

+                goto err;

+        }

+

+        pthread_mutex_lock (&conf->statfs_cache.lock);

+        {

+                *buf = NULL;

+

+                /* Skip if the cache is not initialized */

+                if (!conf->statfs_cache.initialized) {

+                        ret = -1;

+                        goto err;

+                }

+

+                timespec_now (&now;;

+

+                cache_age = (

+                  now.tv_sec - conf->statfs_cache.last_refreshed.tv_sec);

+

+                gf_log (this->name, GF_LOG_DEBUG,

+                        "STATFS cache age = %lf", cache_age);

+                if (cache_age > conf->timeout) {

+                        /* Expire the cache */

+                        gf_log (this->name, GF_LOG_DEBUG,

+                                "Cache age %lf exceeded timeout %d",

+                                cache_age, conf->timeout);

+                        ret = -1;

+                        goto err;

+                }

+

+                *buf = &conf->statfs_cache.buf;

+        }

+err:

+        pthread_mutex_unlock (&conf->statfs_cache.lock);

+        return ret;

+}

+

+int

+mdc_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

+                int32_t op_ret, int32_t op_errno,

+                struct statvfs *buf, dict_t *xdata)

+{

+        struct mdc_conf *conf  = this->private;

+        mdc_local_t     *local = NULL;

+

+        local = frame->local;

+        if (!local)

+                goto out;

+

+        if (op_ret != 0) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE)) {

+                        mdc_inode_iatt_invalidate (this, local->loc.inode);

+                }

+

+                goto out;

+        }

+

+        if (conf && conf->cache_statfs) {

+                mdc_cache_statfs (this, buf);

+        }

+

+out:

+        MDC_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);

+

+        return 0;

+}

+

+int

+mdc_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)

+{

+        int              ret   = 0, op_ret = 0, op_errno = 0;

+        struct statvfs  *buf   = NULL;

+        mdc_local_t     *local = NULL;

+        struct mdc_conf *conf  = this->private;

+

+        local = mdc_local_get (frame);

+        if (!local) {

+                op_ret = -1;

+                op_errno = ENOMEM;

+                goto out;

+        }

+

+        loc_copy (&local->loc, loc);

+

+        if (!conf) {

+                goto uncached;

+        }

+

+        if (!conf->cache_statfs) {

+                goto uncached;

+        }

+

+        ret = mdc_load_statfs_info_from_cache (this, &buf;;

+        if (ret == 0 && buf) {

+                op_ret = 0;

+                op_errno = 0;

+                goto out;

+        }

+

+uncached:

+        STACK_WIND (frame, mdc_statfs_cbk, FIRST_CHILD (this),

+                    FIRST_CHILD (this)->fops->statfs, loc, xdata);

+        return 0;

+

+out:

+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);

+        return 0;

+}

+

+int

+mdc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

+                int32_t       op_ret,	int32_t op_errno, inode_t *inode,

+                struct iatt  *stbuf, dict_t *dict, struct iatt *postparent)

+{

+        mdc_local_t     *local = NULL;

+        struct mdc_conf *conf  = this->private;

         local = frame->local;

         if (op_ret != 0) {

                 if (op_errno == ENOENT)

                         GF_ATOMIC_INC (conf->mdc_counter.negative_lookup);

+

+                if (op_errno == ESTALE) {

+                        /* if op_errno is ENOENT, fuse-bridge will unlink the

+                         * dentry

+                         */

+                        if (local->loc.parent)

+                                mdc_inode_iatt_invalidate (this,

+                                                           local->loc.parent);

+                        else

+                                mdc_inode_iatt_invalidate (this,

+                                                           local->loc.inode);

+                }

+

                 goto out;

         }

@@ -1185,13 +1339,18 @@ mdc_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

 {

         mdc_local_t  *local = NULL;

-        if (op_ret != 0)

-                goto out;

-

         local = frame->local;

         if (!local)

                 goto out;

+        if (op_ret != 0) {

+                if ((op_errno == ESTALE) || (op_errno == ENOENT)) {

+                        mdc_inode_iatt_invalidate (this, local->loc.inode);

+                }

+

+                goto out;

+        }

+

         mdc_inode_iatt_set (this, local->loc.inode, buf);

 out:

@@ -1240,13 +1399,18 @@ mdc_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

 {

         mdc_local_t  *local = NULL;

-        if (op_ret != 0)

-                goto out;

-

         local = frame->local;

         if (!local)

                 goto out;

+        if (op_ret != 0) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE)) {

+                        mdc_inode_iatt_invalidate (this, local->fd->inode);

+                }

+

+                goto out;

+        }

+

         mdc_inode_iatt_set (this, local->fd->inode, buf);

 out:

@@ -1297,11 +1461,15 @@ mdc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         local = frame->local;

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ESTALE) || (op_errno == ENOENT))

+                        mdc_inode_iatt_invalidate (this, local->loc.inode);

+

                 goto out;

+        }

         mdc_inode_iatt_set_validate(this, local->loc.inode, prebuf, postbuf,

                                     _gf_true);

@@ -1340,11 +1508,15 @@ mdc_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         local = frame->local;

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE))

+                        mdc_inode_iatt_invalidate (this, local->fd->inode);

+

                 goto out;

+        }

         mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf,

                                     _gf_true);

@@ -1384,11 +1556,16 @@ mdc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         local = frame->local;

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ESTALE) || (op_errno == ENOENT)) {

+                        mdc_inode_iatt_invalidate (this, local->loc.parent);

+                }

+

                 goto out;

+        }

         if (local->loc.parent) {

                 mdc_inode_iatt_set (this, local->loc.parent, postparent);

@@ -1433,11 +1610,16 @@ mdc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         local = frame->local;

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ESTALE) || (op_errno == ENOENT)) {

+                        mdc_inode_iatt_invalidate (this, local->loc.parent);

+                }

+

                 goto out;

+        }

         if (local->loc.parent) {

                 mdc_inode_iatt_set (this, local->loc.parent, postparent);

@@ -1481,11 +1663,24 @@ mdc_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         local = frame->local;

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                /* if errno is ESTALE, parent is not present, which implies even

+                 * child is not present. Also, man 2 unlink states unlink can

+                 * return ENOENT if a component in pathname does not

+                 * exist or is a dangling symbolic link. So, invalidate both

+                 * parent and child for both errno

+                 */

+

+                if ((op_errno == ENOENT) || (op_errno == ESTALE)) {

+                        mdc_inode_iatt_invalidate (this, local->loc.inode);

+                        mdc_inode_iatt_invalidate (this, local->loc.parent);

+                }

+

                 goto out;

+        }

         if (local->loc.parent) {

                 mdc_inode_iatt_set (this, local->loc.parent, postparent);

@@ -1528,11 +1723,24 @@ mdc_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         local = frame->local;

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                /* if errno is ESTALE, parent is not present, which implies even

+                 * child is not present. Also, man 2 rmdir states rmdir can

+                 * return ENOENT if a directory component in pathname does not

+                 * exist or is a dangling symbolic link. So, invalidate both

+                 * parent and child for both errno

+                 */

+

+                if ((op_errno == ESTALE) || (op_errno == ENOENT)) {

+                        mdc_inode_iatt_invalidate (this, local->loc.inode);

+                        mdc_inode_iatt_invalidate (this, local->loc.parent);

+                }

+

                 goto out;

+        }

         if (local->loc.parent) {

                 mdc_inode_iatt_set (this, local->loc.parent, postparent);

@@ -1572,11 +1780,16 @@ mdc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         local = frame->local;

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ESTALE) || (op_errno == ENOENT)) {

+                        mdc_inode_iatt_invalidate (this, local->loc.parent);

+                }

+

                 goto out;

+        }

         if (local->loc.parent) {

                 mdc_inode_iatt_set (this, local->loc.parent, postparent);

@@ -1621,12 +1834,17 @@ mdc_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         mdc_local_t *local = NULL;

         local = frame->local;

-

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ESTALE) || (op_errno == ENOENT)) {

+                        mdc_inode_iatt_invalidate (this, local->loc.inode);

+                        mdc_inode_iatt_invalidate (this, local->loc2.parent);

+                }

+

                 goto out;

+        }

         if (local->loc.parent) {

                 mdc_inode_iatt_set (this, local->loc.parent, postoldparent);

@@ -1678,11 +1896,17 @@ mdc_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         local = frame->local;

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE)) {

+                        mdc_inode_iatt_invalidate (this, local->loc.inode);

+                        mdc_inode_iatt_invalidate (this, local->loc2.parent);

+                }

+

                 goto out;

+        }

         if (local->loc.inode) {

                 mdc_inode_iatt_set (this, local->loc.inode, buf);

@@ -1726,11 +1950,16 @@ mdc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         local = frame->local;

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ESTALE) || (op_errno == ENOENT)) {

+                        mdc_inode_iatt_invalidate (this, local->loc.parent);

+                }

+

                 goto out;

+        }

         if (local->loc.parent) {

                 mdc_inode_iatt_set (this, local->loc.parent, postparent);

@@ -1774,8 +2003,14 @@ mdc_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         local = frame->local;

-        if (op_ret || !local)

+        if (!local)

+                goto out;

+

+        if (op_ret != 0) {

+                if ((op_errno == ESTALE) || (op_errno == ENOENT))

+                        mdc_inode_iatt_invalidate (this, local->loc.inode);

                 goto out;

+        }

         if (local->fd->flags & O_TRUNC) {

                 /* O_TRUNC modifies file size. Hence invalidate the

@@ -1821,12 +2056,14 @@ mdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         mdc_local_t  *local = NULL;

         local = frame->local;

-

-        if (op_ret < 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret < 0) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE))

+                        mdc_inode_iatt_invalidate (this, local->fd->inode);

                 goto out;

+        }

         mdc_inode_iatt_set (this, local->fd->inode, stbuf);

@@ -1863,12 +2100,14 @@ mdc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         mdc_local_t  *local = NULL;

         local = frame->local;

-

-        if (op_ret == -1)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret == -1) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE))

+                        mdc_inode_iatt_invalidate (this, local->fd->inode);

                 goto out;

+        }

         mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf,

                                     _gf_true);

@@ -1952,12 +2191,14 @@ mdc_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         mdc_local_t  *local = NULL;

         local = frame->local;

-

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ESTALE) || (op_errno == ENOENT))

+                        mdc_inode_iatt_invalidate (this, local->fd->inode);

                 goto out;

+        }

         mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf,

                                     _gf_true);

@@ -1995,12 +2236,14 @@ mdc_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         mdc_local_t  *local = NULL;

         local = frame->local;

-

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE))

+                        mdc_inode_iatt_invalidate (this, local->fd->inode);

                 goto out;

+        }

         mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf,

                                      _gf_true);

@@ -2037,12 +2280,14 @@ mdc_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         mdc_local_t  *local = NULL;

         local = frame->local;

-

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE))

+                        mdc_inode_iatt_invalidate (this, local->loc.inode);

                 goto out;

+        }

         mdc_inode_xatt_update (this, local->loc.inode, local->xattr);

@@ -2080,12 +2325,14 @@ mdc_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         mdc_local_t  *local = NULL;

         local = frame->local;

-

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ESTALE) || (op_errno == ENOENT))

+                        mdc_inode_iatt_invalidate (this, local->fd->inode);

                 goto out;

+        }

         mdc_inode_xatt_update (this, local->fd->inode, local->xattr);

@@ -2121,13 +2368,16 @@ mdc_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

 {

         mdc_local_t  *local = NULL;

-        if (op_ret < 0)

-                goto out;

-

         local = frame->local;

         if (!local)

                 goto out;

+        if (op_ret < 0) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE))

+                        mdc_inode_iatt_invalidate (this, local->loc.inode);

+                goto out;

+        }

+

         mdc_inode_xatt_update (this, local->loc.inode, xattr);

 out:

@@ -2186,13 +2436,16 @@ mdc_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

 {

         mdc_local_t  *local = NULL;

-        if (op_ret < 0)

-                goto out;

-

         local = frame->local;

         if (!local)

                 goto out;

+        if (op_ret < 0) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE))

+                        mdc_inode_iatt_invalidate (this, local->fd->inode);

+                goto out;

+        }

+

         mdc_inode_xatt_update (this, local->fd->inode, xattr);

 out:

@@ -2250,12 +2503,14 @@ mdc_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         mdc_local_t  *local = NULL;

         local = frame->local;

-

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE))

+                        mdc_inode_iatt_invalidate (this, local->loc.inode);

                 goto out;

+        }

 	if (local->key)

 		mdc_inode_xatt_unset (this, local->loc.inode, local->key);

@@ -2319,12 +2574,14 @@ mdc_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

         mdc_local_t  *local = NULL;

         local = frame->local;

-

-        if (op_ret != 0)

+        if (!local)

                 goto out;

-        if (!local)

+        if (op_ret != 0) {

+                if ((op_errno == ENOENT) || (op_errno == ESTALE))

+                        mdc_inode_iatt_invalidate (this, local->fd->inode);

                 goto out;

+        }

 	if (local->key)

 		mdc_inode_xatt_unset (this, local->fd->inode, local->key);

@@ -2379,6 +2636,28 @@ uncached:

         return 0;

 }

+int32_t

+mdc_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,

+                 int32_t op_ret, int32_t op_errno, fd_t *fd,

+                 dict_t *xdata)

+{

+        mdc_local_t *local = NULL;

+

+        local = frame->local;

+        if (!local)

+                goto out;

+

+        if (op_ret == 0)

+                goto out;

+

+        if ((op_errno == ESTALE) || (op_errno == ENOENT))

+                mdc_inode_iatt_invalidate (this, local->loc.inode);

+

+out:

+        MDC_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd, xdata);

+        return 0;

+}

+

 int

 mdc_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc,

@@ -2387,6 +2666,11 @@ mdc_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc,

         int          ret = -1;

         char        *mdc_key_names = NULL;

         dict_t      *xattr_alloc = NULL;

+        mdc_local_t *local       = NULL;

+

+        local = mdc_local_get (frame);