From 0a3577c161bc1e23c8d4f4a70dff4c1cc5a2c363 Mon Sep 17 00:00:00 2001

From: Niels de Vos <ndevos@redhat.com>

Date: Thu, 6 Apr 2017 15:11:00 +0200

Subject: [PATCH 615/616] io-stats: use gf_atomic_t instead of partial atomic

 variables

io-stats should not use the legacy __sync_*() builtin functions for

doing atomic operations. Instead, it should use the gf_atomic_t type and

macros for any of the statistics it calculates. This makes sure that the

behaviour is the same on all architectures.

Also the __sync_*() builtins are being deprecated with __atomic_*()

functions. The "atomic.h" header will be one of the very few places

where these builtin functions are used and the feature checking will be

needed.

While replacing many of the uint64_t types, it seemed that locking

around some of the statements is not needed anymore (done automatically

with the GF_ATOMIC_*() macros). This resulted in quite some removal and

cleanup of those BUMP_*() macros. It seemed appropriate to make these

macros normal functions and let the compiler decide on inlining them.

This caused some existing functions to be shuffled around.

>Reviewed-on: https://review.gluster.org/17009

>URL: https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html

>Smoke: Gluster Build System <jenkins@build.gluster.org>

>NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

>CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

>Reviewed-by: Amar Tumballi <amarts@redhat.com>

>Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>

Change-Id: I7c4f3df1832eae51debda2b127b943e14546b605

BUG: 1475721

Signed-off-by: Niels de Vos <ndevos@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/117185

Tested-by: Atin Mukherjee <amukherj@redhat.com>

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

---

 xlators/debug/io-stats/src/io-stats.c | 747 ++++++++++++++++++----------------

 1 file changed, 392 insertions(+), 355 deletions(-)

diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c

index 46e1272..92f05e5 100644

--- a/xlators/debug/io-stats/src/io-stats.c

+++ b/xlators/debug/io-stats/src/io-stats.c

@@ -42,6 +42,7 @@

 #define MAX_LIST_MEMBERS 100

 #define DEFAULT_PWD_BUF_SZ 16384

 #define DEFAULT_GRP_BUF_SZ 16384

+#define IOS_BLOCK_COUNT_SIZE 32

 typedef enum {

         IOS_STATS_TYPE_NONE,

@@ -70,7 +71,7 @@ struct ios_stat {

         gf_lock_t       lock;

         uuid_t          gfid;

         char           *filename;

-        uint64_t        counters [IOS_STATS_TYPE_MAX];

+        gf_atomic_t     counters[IOS_STATS_TYPE_MAX];

         struct ios_stat_lat thru_counters [IOS_STATS_THRU_MAX];

         int             refcnt;

 };

@@ -115,12 +116,12 @@ struct ios_lat {

 };

 struct ios_global_stats {

-        uint64_t        data_written;

-        uint64_t        data_read;

-        uint64_t        block_count_write[32];

-        uint64_t        block_count_read[32];

-        uint64_t        fop_hits[GF_FOP_MAXVALUE];

-        uint64_t        upcall_hits[GF_UPCALL_FLAGS_MAXVALUE];

+        gf_atomic_t     data_written;

+        gf_atomic_t     data_read;

+        gf_atomic_t     block_count_write[IOS_BLOCK_COUNT_SIZE];

+        gf_atomic_t     block_count_read[IOS_BLOCK_COUNT_SIZE];

+        gf_atomic_t     fop_hits[GF_FOP_MAXVALUE];

+        gf_atomic_t     upcall_hits[GF_UPCALL_FLAGS_MAXVALUE];

         struct timeval  started_at;

         struct ios_lat  latency[GF_FOP_MAXVALUE];

         uint64_t        nr_opens;

@@ -164,10 +165,10 @@ struct ios_conf {

 struct ios_fd {

         char           *filename;

-        uint64_t        data_written;

-        uint64_t        data_read;

-        uint64_t        block_count_write[32];

-        uint64_t        block_count_read[32];

+        gf_atomic_t     data_written;

+        gf_atomic_t     data_read;

+        gf_atomic_t     block_count_write[IOS_BLOCK_COUNT_SIZE];

+        gf_atomic_t     block_count_read[IOS_BLOCK_COUNT_SIZE];

         struct timeval  opened_at;

 };

@@ -244,26 +245,10 @@ is_fop_latency_started (call_frame_t *frame)

                 conf = this->private;                                   \

                 if (!conf)                                              \

                         break;                                          \

-                conf->cumulative.fop_hits[GF_FOP_##op]++;               \

-                conf->incremental.fop_hits[GF_FOP_##op]++;              \

+                GF_ATOMIC_INC (conf->cumulative.fop_hits[GF_FOP_##op]); \

+                GF_ATOMIC_INC (conf->incremental.fop_hits[GF_FOP_##op]);\

         } while (0)

-#if defined(HAVE_SYNC_BUILTINS)

-/* FIXME: use gf_atomic_t from libglusterfs/src/atomic.h

- *

- * This is currently not behaving correctly. Values are going out of sync in

- * the case where HAVE_SYNC_BUILTINS are available, but are updated under a

- * single lock for other cases.

- */

-#define STATS_LOCK(x)

-#define STATS_UNLOCK(x)

-#define STATS_ADD(x,i)  __sync_add_and_fetch (&x, i)

-#else

-#define STATS_LOCK(x)   LOCK (x)

-#define STATS_UNLOCK(x) UNLOCK (x)

-#define STATS_ADD(x,i)  (x) += (i)

-#endif

-

 #define UPDATE_PROFILE_STATS(frame, op)                                       \

         do {                                                                  \

                 struct ios_conf  *conf = NULL;                                \

@@ -271,85 +256,12 @@ is_fop_latency_started (call_frame_t *frame)

                 if (!is_fop_latency_started (frame))                          \

                         break;                                                \

                 conf = this->private;                                         \

-                STATS_LOCK (&conf->lock);                                     \

-                {                                                             \

-                        if (conf && conf->measure_latency &&                  \

-                            conf->count_fop_hits) {                           \

-                                BUMP_FOP(op);                                 \

-                                gettimeofday (&frame->end, NULL);             \

-                                update_ios_latency (conf, frame, GF_FOP_##op);\

-                        }                                                     \

+                if (conf && conf->measure_latency &&                          \

+                    conf->count_fop_hits) {                                   \

+                        BUMP_FOP (op);                                        \

+                        gettimeofday (&frame->end, NULL);                     \

+                        update_ios_latency (conf, frame, GF_FOP_##op);        \

                 }                                                             \

-                STATS_UNLOCK (&conf->lock);                                   \

-        } while (0)

-

-#define BUMP_READ(fd, len)                                                     \

-        do {                                                                   \

-                struct ios_conf  *conf = NULL;                                 \

-                struct ios_fd    *iosfd = NULL;                                \

-                int               lb2 = 0;                                     \

-                                                                               \

-                conf = this->private;                                          \

-                lb2 = log_base2 (len);                                         \

-                ios_fd_ctx_get (fd, this, &iosfd);                             \

-                if (!conf)                                                     \

-                        break;                                                 \

-                                                                               \

-                STATS_LOCK (&conf->lock);                                      \

-                {                                                              \

-                        STATS_ADD (conf->cumulative.data_read, len);           \

-                        STATS_ADD (conf->incremental.data_read, len);          \

-                        STATS_ADD (conf->cumulative.block_count_read[lb2], 1); \

-                        STATS_ADD (conf->incremental.block_count_read[lb2], 1);\

-                                                                               \

-                        if (iosfd) {                                           \

-                                STATS_ADD (iosfd->data_read, len);             \

-                                STATS_ADD (iosfd->block_count_read[lb2], 1);   \

-                        }                                                      \

-                }                                                              \

-                STATS_UNLOCK (&conf->lock);                                    \

-        } while (0)

-

-#define BUMP_WRITE(fd, len)                                                    \

-        do {                                                                   \

-                struct ios_conf  *conf = NULL;                                 \

-                struct ios_fd    *iosfd = NULL;                                \

-                int               lb2 = 0;                                     \

-                                                                               \

-                conf = this->private;                                          \

-                lb2 = log_base2 (len);                                         \

-                ios_fd_ctx_get (fd, this, &iosfd);                             \

-                if (!conf)                                                     \

-                        break;                                                 \

-                STATS_LOCK (&conf->lock);                                      \

-                {                                                              \

-                        STATS_ADD (conf->cumulative.data_written, len);        \

-                        STATS_ADD (conf->incremental.data_written, len);       \

-                        STATS_ADD (conf->cumulative.block_count_write[lb2], 1);\

-                        STATS_ADD (conf->incremental.block_count_write[lb2], 1);\

-                                                                               \

-                        if (iosfd) {                                           \

-                                STATS_ADD (iosfd->data_written, len);          \

-                                STATS_ADD (iosfd->block_count_write[lb2], 1);  \

-                        }                                                      \

-                }                                                              \

-                STATS_UNLOCK (&conf->lock);                                    \

-        } while (0)

-

-#define BUMP_STATS(iosstat, type)                                       \

-        do {                                                            \

-                struct ios_conf         *conf = NULL;                   \

-                uint64_t                 value = 0;                     \

-                                                                        \

-                conf = this->private;                                   \

-                                                                        \

-                LOCK(&iosstat->lock);                                   \

-                {                                                       \

-                        value = STATS_ADD (iosstat->counters[type], 1); \

-                }                                                       \

-                UNLOCK (&iosstat->lock);                                \

-                ios_stat_add_to_list (&conf->list[type],                \

-                                     value, iosstat);                   \

         } while (0)

 #define BUMP_THROUGHPUT(iosstat, type)                                         \

@@ -368,7 +280,7 @@ is_fop_latency_started (call_frame_t *frame)

                 throughput = op_ret / elapsed;                                 \

                                                                                \

                 conf = this->private;                                          \

-                STATS_LOCK (&iosstat->lock);                                   \

+                LOCK (&iosstat->lock);                                         \

                 {                                                              \

                         if (iosstat->thru_counters[type].throughput            \

                                 <= throughput) {                               \

@@ -379,26 +291,14 @@ is_fop_latency_started (call_frame_t *frame)

                                flag = 1;                                       \

                         }                                                      \

                 }                                                              \

-                STATS_UNLOCK (&iosstat->lock);                                 \

+                UNLOCK (&iosstat->lock);                                       \

                 if (flag)                                                      \

                        ios_stat_add_to_list (&conf->thru_list[type],           \

                                                throughput, iosstat);           \

         } while (0)

-#define BUMP_UPCALL(event)                                                     \

-        do {                                                                   \

-                struct ios_conf  *conf = NULL;                                 \

-                                                                               \

-                conf = this->private;                                          \

-                if (!conf)                                                     \

-                        break;                                                 \

-                if (conf->count_fop_hits) {                                    \

-                        conf->cumulative.upcall_hits[event]++;                 \

-                        conf->incremental.upcall_hits[event]++;                \

-                }                                                              \

-        } while (0)

-int

+static int

 ios_fd_ctx_get (fd_t *fd, xlator_t *this, struct ios_fd **iosfd)

 {

         uint64_t      iosfd64 = 0;

@@ -414,8 +314,7 @@ ios_fd_ctx_get (fd_t *fd, xlator_t *this, struct ios_fd **iosfd)

 }

-

-int

+static int

 ios_fd_ctx_set (fd_t *fd, xlator_t *this, struct ios_fd *iosfd)

 {

         uint64_t   iosfd64 = 0;

@@ -427,7 +326,8 @@ ios_fd_ctx_set (fd_t *fd, xlator_t *this, struct ios_fd *iosfd)

         return ret;

 }

-int

+

+static int

 ios_stat_ref (struct ios_stat *iosstat)

 {

         LOCK (&iosstat->lock);

@@ -439,7 +339,8 @@ ios_stat_ref (struct ios_stat *iosstat)

         return iosstat->refcnt;

 }

-int

+

+static int

 ios_stat_unref (struct ios_stat *iosstat)

 {

         int cleanup = 0;

@@ -465,101 +366,8 @@ ios_stat_unref (struct ios_stat *iosstat)

         return 0;

 }

-int

-ios_inode_ctx_set (inode_t *inode, xlator_t *this, struct ios_stat *iosstat)

-{

-        uint64_t   iosstat64 = 0;

-        int        ret     = 0;

-

-        ios_stat_ref (iosstat);

-        iosstat64 = (unsigned long )iosstat;

-        ret = inode_ctx_put (inode, this, iosstat64);

-        return ret;

-}

-

-int

-ios_inode_ctx_get (inode_t *inode, xlator_t *this, struct ios_stat **iosstat)

-{

-        uint64_t      iosstat64 = 0;

-        unsigned long iosstatlong = 0;

-        int           ret = 0;

-

-        ret = inode_ctx_get (inode, this, &iosstat64);

-        iosstatlong = iosstat64;

-        if (ret != -1)

-                *iosstat = (void *) iosstatlong;

-

-        return ret;

-

-}

-

-/*

- * So why goto all this trouble?  Why not just queue up some samples in

- * a big list and malloc away?  Well malloc is expensive relative

- * to what we are measuring, so cannot have any malloc's (or worse

- * callocs) in our measurement code paths.  Instead, we are going to

- * pre-allocate a circular buffer and collect a maximum number of samples.

- * Prior to dumping them all we'll create a new buffer and swap the

- * old buffer with the new, and then proceed to dump the statistics

- * in our dump thread.

- *

- */

-ios_sample_buf_t *

-ios_create_sample_buf (size_t buf_size)

-{

-        ios_sample_buf_t *ios_sample_buf = NULL;

-        ios_sample_t     *ios_samples = NULL;

-

-        ios_sample_buf = GF_CALLOC (1,

-                sizeof (*ios_sample_buf),

-                gf_io_stats_mt_ios_sample_buf);

-        if (!ios_sample_buf)

-                goto err;

-

-        ios_samples = GF_CALLOC (buf_size,

-                sizeof (*ios_samples),

-                gf_io_stats_mt_ios_sample);

-

-        if (!ios_samples)

-                goto err;

-

-        ios_sample_buf->ios_samples = ios_samples;

-        ios_sample_buf->size = buf_size;

-        ios_sample_buf->pos = 0;

-        ios_sample_buf->observed = 0;

-        ios_sample_buf->collected = 0;

-

-        return ios_sample_buf;

-err:

-        GF_FREE (ios_sample_buf);

-        return NULL;

-}

-

-void

-ios_destroy_sample_buf (ios_sample_buf_t *ios_sample_buf)

-{

-        GF_FREE (ios_sample_buf->ios_samples);

-        GF_FREE (ios_sample_buf);

-}

 static int

-ios_init_sample_buf (struct ios_conf *conf)

-{

-        int32_t        ret = -1;

-

-        GF_ASSERT (conf);

-        LOCK (&conf->lock);

-        conf->ios_sample_buf = ios_create_sample_buf (

-                conf->ios_sample_buf_size);

-        if (!conf->ios_sample_buf)

-                goto out;

-        ret = 0;

-out:

-        UNLOCK (&conf->lock);

-        return ret;

-}

-

-int

 ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,

                             struct ios_stat *iosstat)

 {

@@ -654,6 +462,180 @@ out:

         return 0;

 }

+

+static void

+ios_bump_read (xlator_t *this, fd_t *fd, size_t len)

+{

+        struct ios_conf  *conf = NULL;

+        struct ios_fd    *iosfd = NULL;

+        int               lb2 = 0;

+

+        conf = this->private;

+        lb2 = log_base2 (len);

+        ios_fd_ctx_get (fd, this, &iosfd);

+        if (!conf)

+                return;

+

+        GF_ATOMIC_ADD (conf->cumulative.data_read, len);

+        GF_ATOMIC_ADD (conf->incremental.data_read, len);

+        GF_ATOMIC_INC (conf->cumulative.block_count_read[lb2]);

+        GF_ATOMIC_INC (conf->incremental.block_count_read[lb2]);

+

+        if (iosfd) {

+                GF_ATOMIC_ADD (iosfd->data_read, len);

+                GF_ATOMIC_INC (iosfd->block_count_read[lb2]);

+        }

+}

+

+

+static void

+ios_bump_write (xlator_t *this, fd_t *fd, size_t len)

+{

+        struct ios_conf  *conf = NULL;

+        struct ios_fd    *iosfd = NULL;

+        int               lb2 = 0;

+

+        conf = this->private;

+        lb2 = log_base2 (len);

+        ios_fd_ctx_get (fd, this, &iosfd);

+        if (!conf)

+                return;

+

+        GF_ATOMIC_ADD (conf->cumulative.data_written, len);

+        GF_ATOMIC_ADD (conf->incremental.data_written, len);

+        GF_ATOMIC_INC (conf->cumulative.block_count_write[lb2]);

+        GF_ATOMIC_INC (conf->incremental.block_count_write[lb2]);

+

+        if (iosfd) {

+                GF_ATOMIC_ADD (iosfd->data_written, len);

+                GF_ATOMIC_INC (iosfd->block_count_write[lb2]);

+        }

+}

+

+

+static void

+ios_bump_upcall (xlator_t *this, gf_upcall_flags_t event)

+{

+        struct ios_conf  *conf = NULL;

+

+        conf = this->private;

+        if (!conf)

+                return;

+        if (conf->count_fop_hits) {

+                GF_ATOMIC_INC (conf->cumulative.upcall_hits[event]);

+                GF_ATOMIC_INC (conf->incremental.upcall_hits[event]);

+        }

+}

+

+

+static void

+ios_bump_stats (xlator_t *this, struct ios_stat *iosstat,

+                ios_stats_type_t type)

+{

+        struct ios_conf         *conf = NULL;

+        uint64_t                 value = 0;

+

+        conf = this->private;

+

+        value = GF_ATOMIC_INC (iosstat->counters[type]);

+        ios_stat_add_to_list (&conf->list[type], value, iosstat);

+}

+

+

+int

+ios_inode_ctx_set (inode_t *inode, xlator_t *this, struct ios_stat *iosstat)

+{

+        uint64_t   iosstat64 = 0;

+        int        ret     = 0;

+

+        ios_stat_ref (iosstat);

+        iosstat64 = (unsigned long) iosstat;

+        ret = inode_ctx_put (inode, this, iosstat64);

+        return ret;

+}

+

+int

+ios_inode_ctx_get (inode_t *inode, xlator_t *this, struct ios_stat **iosstat)

+{

+        uint64_t      iosstat64 = 0;

+        unsigned long iosstatlong = 0;

+        int           ret = 0;

+

+        ret = inode_ctx_get (inode, this, &iosstat64);

+        iosstatlong = iosstat64;

+        if (ret != -1)

+                *iosstat = (void *) iosstatlong;

+

+        return ret;

+

+}

+

+/*

+ * So why goto all this trouble?  Why not just queue up some samples in

+ * a big list and malloc away?  Well malloc is expensive relative

+ * to what we are measuring, so cannot have any malloc's (or worse

+ * callocs) in our measurement code paths.  Instead, we are going to

+ * pre-allocate a circular buffer and collect a maximum number of samples.

+ * Prior to dumping them all we'll create a new buffer and swap the

+ * old buffer with the new, and then proceed to dump the statistics

+ * in our dump thread.

+ *

+ */

+ios_sample_buf_t *

+ios_create_sample_buf (size_t buf_size)

+{

+        ios_sample_buf_t *ios_sample_buf = NULL;

+        ios_sample_t     *ios_samples = NULL;

+

+        ios_sample_buf = GF_CALLOC (1,

+                sizeof (*ios_sample_buf),

+                gf_io_stats_mt_ios_sample_buf);

+        if (!ios_sample_buf)

+                goto err;

+

+        ios_samples = GF_CALLOC (buf_size,

+                sizeof (*ios_samples),

+                gf_io_stats_mt_ios_sample);

+

+        if (!ios_samples)

+                goto err;

+

+        ios_sample_buf->ios_samples = ios_samples;

+        ios_sample_buf->size = buf_size;

+        ios_sample_buf->pos = 0;

+        ios_sample_buf->observed = 0;

+        ios_sample_buf->collected = 0;

+

+        return ios_sample_buf;

+err:

+        GF_FREE (ios_sample_buf);

+        return NULL;

+}

+

+void

+ios_destroy_sample_buf (ios_sample_buf_t *ios_sample_buf)

+{

+        GF_FREE (ios_sample_buf->ios_samples);

+        GF_FREE (ios_sample_buf);

+}

+

+static int

+ios_init_sample_buf (struct ios_conf *conf)

+{

+        int32_t        ret = -1;

+

+        GF_ASSERT (conf);

+        LOCK (&conf->lock);

+        conf->ios_sample_buf = ios_create_sample_buf (

+                conf->ios_sample_buf_size);

+        if (!conf->ios_sample_buf)

+                goto out;

+        ret = 0;

+out:

+        UNLOCK (&conf->lock);

+        return ret;

+}

+

 static int

 ios_stats_cleanup (xlator_t *this, inode_t *inode)

 {

@@ -808,7 +790,7 @@ io_stats_dump_global_to_json_logfp (xlator_t *this,

         int                   ret = 1;  /* Default to error */

         int                   rw_size;

         char                  *rw_unit = NULL;

-        long                  fop_hits;

+        uint64_t              fop_hits;

         float                 fop_lat_ave;

         float                 fop_lat_min;

         float                 fop_lat_max;

@@ -847,24 +829,26 @@ io_stats_dump_global_to_json_logfp (xlator_t *this,

                 if (interval == -1) {

                         ios_log (this, logfp,

-                                "\"%s.%s.read_%d%s\": \"%"PRId64"\",",

+                                "\"%s.%s.read_%d%s\": \"%"GF_PRI_ATOMIC"\",",

                                 key_prefix, str_prefix, rw_size, rw_unit,

-                                stats->block_count_read[i]);

+                                GF_ATOMIC_GET (stats->block_count_read[i]));

                         ios_log (this, logfp,

-                                "\"%s.%s.write_%d%s\": \"%"PRId64"\",",

+                                "\"%s.%s.write_%d%s\": \"%"GF_PRI_ATOMIC"\",",

                                 key_prefix, str_prefix, rw_size, rw_unit,

-                                stats->block_count_write[i]);

+                                GF_ATOMIC_GET (stats->block_count_write[i]));

                 } else {

                         ios_log (this, logfp,

                                 "\"%s.%s.read_%d%s_per_sec\": \"%0.2lf\",",

                                 key_prefix, str_prefix, rw_size, rw_unit,

-                                (double)(stats->block_count_read[i] /

-                                        interval_sec));

+                                (double)

+                                (GF_ATOMIC_GET (stats->block_count_read[i]) /

+                                 interval_sec));

                         ios_log (this, logfp,

                                 "\"%s.%s.write_%d%s_per_sec\": \"%0.2lf\",",

                                 key_prefix, str_prefix, rw_size, rw_unit,

-                                (double)(stats->block_count_write[i] /

-                                        interval_sec));

+                                (double)

+                                (GF_ATOMIC_GET (stats->block_count_write[i]) /

+                                 interval_sec));

                 }

         }

@@ -883,12 +867,11 @@ io_stats_dump_global_to_json_logfp (xlator_t *this,

                         lc_fop_name[j] = tolower (lc_fop_name[j]);

                 }

-                fop_hits = 0;

+                fop_hits = GF_ATOMIC_GET (stats->fop_hits[i]);

                 fop_lat_ave = 0.0;

                 fop_lat_min = 0.0;

                 fop_lat_max = 0.0;

-                if (stats->fop_hits[i]) {

-                        fop_hits = stats->fop_hits[i];

+                if (fop_hits) {

                         if (stats->latency[i].avg) {

                                 fop_lat_ave = stats->latency[i].avg;

                                 fop_lat_min = stats->latency[i].min;

@@ -897,7 +880,7 @@ io_stats_dump_global_to_json_logfp (xlator_t *this,

                 }

                 if (interval == -1) {

                         ios_log (this, logfp,

-                                "\"%s.%s.fop.%s.count\": \"%"PRId64"\",",

+                                "\"%s.%s.fop.%s.count\": \"%"GF_PRI_ATOMIC"\",",

                                 key_prefix, str_prefix, lc_fop_name,

                                 fop_hits);

                 } else {

@@ -923,10 +906,10 @@ io_stats_dump_global_to_json_logfp (xlator_t *this,

                 for (j = 0; lc_fop_name[j]; j++) {

                         lc_fop_name[j] = tolower (lc_fop_name[j]);

                 }

-                fop_hits = stats->upcall_hits[i];

+                fop_hits = GF_ATOMIC_GET (stats->upcall_hits[i]);

                 if (interval == -1) {

                         ios_log (this, logfp,

-                                "\"%s.%s.fop.%s.count\": \"%"PRId64"\",",

+                                "\"%s.%s.fop.%s.count\": \"%"GF_PRI_ATOMIC"\",",

                                 key_prefix, str_prefix, lc_fop_name,

                                 fop_hits);

                 } else {

@@ -941,10 +924,12 @@ io_stats_dump_global_to_json_logfp (xlator_t *this,

                 ios_log (this, logfp, "\"%s.%s.uptime\": \"%"PRId64"\",",

                          key_prefix, str_prefix,

                          (uint64_t) (now->tv_sec - stats->started_at.tv_sec));

-                ios_log (this, logfp, "\"%s.%s.bytes_read\": \"%"PRId64"\",",

-                         key_prefix, str_prefix, stats->data_read);

-                ios_log (this, logfp, "\"%s.%s.bytes_written\": \"%"PRId64"\"",

-                         key_prefix, str_prefix, stats->data_written);

+                ios_log (this, logfp, "\"%s.%s.bytes_read\": \""

+                         "%"GF_PRI_ATOMIC"\",", key_prefix, str_prefix,

+                         GF_ATOMIC_GET (stats->data_read));

+                ios_log (this, logfp, "\"%s.%s.bytes_written\": \""

+                         "%"GF_PRI_ATOMIC"\"", key_prefix, str_prefix,

+                         GF_ATOMIC_GET (stats->data_written));

         } else {

                 ios_log (this, logfp,

                          "\"%s.%s.sample_interval_sec\": \"%0.2lf\",",

@@ -952,12 +937,12 @@ io_stats_dump_global_to_json_logfp (xlator_t *this,

                          interval_sec);

                 ios_log (this, logfp,

                          "\"%s.%s.bytes_read_per_sec\": \"%0.2lf\",",

-                         key_prefix, str_prefix,

-                         (double)(stats->data_read / interval_sec));

+                         key_prefix, str_prefix, (double)

+                         (GF_ATOMIC_GET (stats->data_read) / interval_sec));

                 ios_log (this, logfp,

                          "\"%s.%s.bytes_written_per_sec\": \"%0.2lf\"",

-                         key_prefix, str_prefix,

-                         (double)(stats->data_written / interval_sec));

+                         key_prefix, str_prefix, (double)

+                         (GF_ATOMIC_GET (stats->data_written) / interval_sec));

         }

         ios_log (this, logfp, "}");

@@ -1204,6 +1189,9 @@ io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,

         char                  str_header[128] = {0};

         char                  str_read[128] = {0};

         char                  str_write[128] = {0};

+        uint64_t              fop_hits = 0;

+        uint64_t              block_count_read = 0;

+        uint64_t              block_count_write = 0;

         conf = this->private;

@@ -1214,31 +1202,32 @@ io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,

                          interval);

         ios_log (this, logfp, "      Duration : %"PRId64" secs",

                  (uint64_t) (now->tv_sec - stats->started_at.tv_sec));

-        ios_log (this, logfp, "     BytesRead : %"PRId64,

-                 stats->data_read);

-        ios_log (this, logfp, "  BytesWritten : %"PRId64"\n",

-                 stats->data_written);

+        ios_log (this, logfp, "     BytesRead : %"GF_PRI_ATOMIC,

+                 GF_ATOMIC_GET (stats->data_read));

+        ios_log (this, logfp, "  BytesWritten : %"GF_PRI_ATOMIC"\n",

+                 GF_ATOMIC_GET (stats->data_written));

         snprintf (str_header, sizeof (str_header), "%-12s %c", "Block Size", ':');

         snprintf (str_read, sizeof (str_read), "%-12s %c", "Read Count", ':');

         snprintf (str_write, sizeof (str_write), "%-12s %c", "Write Count", ':');

         index = 14;

-        for (i = 0; i < 32; i++) {

-                if ((stats->block_count_read[i] == 0) &&

-                    (stats->block_count_write[i] == 0))

+        for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {

+                block_count_read = GF_ATOMIC_GET (stats->block_count_read[i]);

+                block_count_write = GF_ATOMIC_GET (stats->block_count_write[i]);

+                if ((block_count_read == 0) && (block_count_write == 0))

                         continue;