9f5ccc
From f199094cb61341a47c98a8ed91b293446182b5a9 Mon Sep 17 00:00:00 2001
9f5ccc
From: Mohit Agrawal <moagrawal@redhat.com>
9f5ccc
Date: Thu, 3 Oct 2019 14:06:52 +0530
9f5ccc
Subject: [PATCH 333/335] rpc: Synchronize slot allocation code
9f5ccc
9f5ccc
Problem: Current slot allocation/deallocation code path is not
9f5ccc
         synchronized.There are scenario when due to race condition
9f5ccc
         in slot allocation/deallocation code path brick is crashed.
9f5ccc
9f5ccc
Solution: Synchronize slot allocation/deallocation code path to
9f5ccc
          avoid the issue
9f5ccc
9f5ccc
> Change-Id: I4fb659a75234218ffa0e5e0bf9308f669f75fc25
9f5ccc
> Fixes: bz#1763036
9f5ccc
> Signed-off-by: Mohit Agrawal <moagrawal@redhat.com>
9f5ccc
> (Reviewed on upstream link https://review.gluster.org/#/c/glusterfs/+/23508/)
9f5ccc
> (Cherry pick from commit faf5ac13c4ee00a05e9451bf8da3be2a9043bbf2)
9f5ccc
9f5ccc
Change-Id: I4fb659a75234218ffa0e5e0bf9308f669f75fc25
9f5ccc
BUG: 1741193
9f5ccc
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>
9f5ccc
Reviewed-on: https://code.engineering.redhat.com/gerrit/185827
9f5ccc
Tested-by: RHGS Build Bot <nigelb@redhat.com>
9f5ccc
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
9f5ccc
---
9f5ccc
 libglusterfs/src/event-epoll.c | 74 +++++++++++++++++++++++-------------------
9f5ccc
 1 file changed, 41 insertions(+), 33 deletions(-)
9f5ccc
9f5ccc
diff --git a/libglusterfs/src/event-epoll.c b/libglusterfs/src/event-epoll.c
9f5ccc
index 0cec47e..65f5efd 100644
9f5ccc
--- a/libglusterfs/src/event-epoll.c
9f5ccc
+++ b/libglusterfs/src/event-epoll.c
9f5ccc
@@ -69,15 +69,27 @@ __event_newtable(struct event_pool *event_pool, int table_idx)
9f5ccc
 }
9f5ccc
 
9f5ccc
 static int
9f5ccc
+event_slot_ref(struct event_slot_epoll *slot)
9f5ccc
+{
9f5ccc
+    if (!slot)
9f5ccc
+        return -1;
9f5ccc
+
9f5ccc
+    return GF_ATOMIC_INC(slot->ref);
9f5ccc
+}
9f5ccc
+
9f5ccc
+static int
9f5ccc
 __event_slot_alloc(struct event_pool *event_pool, int fd,
9f5ccc
-                   char notify_poller_death)
9f5ccc
+                   char notify_poller_death, struct event_slot_epoll **slot)
9f5ccc
 {
9f5ccc
     int i = 0;
9f5ccc
+    int j = 0;
9f5ccc
     int table_idx = -1;
9f5ccc
     int gen = -1;
9f5ccc
     struct event_slot_epoll *table = NULL;
9f5ccc
 
9f5ccc
-    for (i = 0; i < EVENT_EPOLL_TABLES; i++) {
9f5ccc
+retry:
9f5ccc
+
9f5ccc
+    while (i < EVENT_EPOLL_TABLES) {
9f5ccc
         switch (event_pool->slots_used[i]) {
9f5ccc
             case EVENT_EPOLL_SLOTS:
9f5ccc
                 continue;
9f5ccc
@@ -98,6 +110,7 @@ __event_slot_alloc(struct event_pool *event_pool, int fd,
9f5ccc
         if (table)
9f5ccc
             /* break out of the loop */
9f5ccc
             break;
9f5ccc
+        i++;
9f5ccc
     }
9f5ccc
 
9f5ccc
     if (!table)
9f5ccc
@@ -105,20 +118,20 @@ __event_slot_alloc(struct event_pool *event_pool, int fd,
9f5ccc
 
9f5ccc
     table_idx = i;
9f5ccc
 
9f5ccc
-    for (i = 0; i < EVENT_EPOLL_SLOTS; i++) {
9f5ccc
-        if (table[i].fd == -1) {
9f5ccc
+    for (j = 0; j < EVENT_EPOLL_SLOTS; j++) {
9f5ccc
+        if (table[j].fd == -1) {
9f5ccc
             /* wipe everything except bump the generation */
9f5ccc
-            gen = table[i].gen;
9f5ccc
-            memset(&table[i], 0, sizeof(table[i]));
9f5ccc
-            table[i].gen = gen + 1;
9f5ccc
+            gen = table[j].gen;
9f5ccc
+            memset(&table[j], 0, sizeof(table[j]));
9f5ccc
+            table[j].gen = gen + 1;
9f5ccc
 
9f5ccc
-            LOCK_INIT(&table[i].lock);
9f5ccc
-            INIT_LIST_HEAD(&table[i].poller_death);
9f5ccc
+            LOCK_INIT(&table[j].lock);
9f5ccc
+            INIT_LIST_HEAD(&table[j].poller_death);
9f5ccc
 
9f5ccc
-            table[i].fd = fd;
9f5ccc
+            table[j].fd = fd;
9f5ccc
             if (notify_poller_death) {
9f5ccc
-                table[i].idx = table_idx * EVENT_EPOLL_SLOTS + i;
9f5ccc
-                list_add_tail(&table[i].poller_death,
9f5ccc
+                table[j].idx = table_idx * EVENT_EPOLL_SLOTS + j;
9f5ccc
+                list_add_tail(&table[j].poller_death,
9f5ccc
                               &event_pool->poller_death);
9f5ccc
             }
9f5ccc
 
9f5ccc
@@ -128,18 +141,26 @@ __event_slot_alloc(struct event_pool *event_pool, int fd,
9f5ccc
         }
9f5ccc
     }
9f5ccc
 
9f5ccc
-    return table_idx * EVENT_EPOLL_SLOTS + i;
9f5ccc
+    if (j == EVENT_EPOLL_SLOTS) {
9f5ccc
+        table = NULL;
9f5ccc
+        i++;
9f5ccc
+        goto retry;
9f5ccc
+    } else {
9f5ccc
+        (*slot) = &table[j];
9f5ccc
+        event_slot_ref(*slot);
9f5ccc
+        return table_idx * EVENT_EPOLL_SLOTS + j;
9f5ccc
+    }
9f5ccc
 }
9f5ccc
 
9f5ccc
 static int
9f5ccc
 event_slot_alloc(struct event_pool *event_pool, int fd,
9f5ccc
-                 char notify_poller_death)
9f5ccc
+                 char notify_poller_death, struct event_slot_epoll **slot)
9f5ccc
 {
9f5ccc
     int idx = -1;
9f5ccc
 
9f5ccc
     pthread_mutex_lock(&event_pool->mutex);
9f5ccc
     {
9f5ccc
-        idx = __event_slot_alloc(event_pool, fd, notify_poller_death);
9f5ccc
+        idx = __event_slot_alloc(event_pool, fd, notify_poller_death, slot);
9f5ccc
     }
9f5ccc
     pthread_mutex_unlock(&event_pool->mutex);
9f5ccc
 
9f5ccc
@@ -153,6 +174,7 @@ __event_slot_dealloc(struct event_pool *event_pool, int idx)
9f5ccc
     int offset = 0;
9f5ccc
     struct event_slot_epoll *table = NULL;
9f5ccc
     struct event_slot_epoll *slot = NULL;
9f5ccc
+    int fd = -1;
9f5ccc
 
9f5ccc
     table_idx = idx / EVENT_EPOLL_SLOTS;
9f5ccc
     offset = idx % EVENT_EPOLL_SLOTS;
9f5ccc
@@ -164,11 +186,13 @@ __event_slot_dealloc(struct event_pool *event_pool, int idx)
9f5ccc
     slot = &table[offset];
9f5ccc
     slot->gen++;
9f5ccc
 
9f5ccc
+    fd = slot->fd;
9f5ccc
     slot->fd = -1;
9f5ccc
     slot->handled_error = 0;
9f5ccc
     slot->in_handler = 0;
9f5ccc
     list_del_init(&slot->poller_death);
9f5ccc
-    event_pool->slots_used[table_idx]--;
9f5ccc
+    if (fd != -1)
9f5ccc
+        event_pool->slots_used[table_idx]--;
9f5ccc
 
9f5ccc
     return;
9f5ccc
 }
9f5ccc
@@ -185,15 +209,6 @@ event_slot_dealloc(struct event_pool *event_pool, int idx)
9f5ccc
     return;
9f5ccc
 }
9f5ccc
 
9f5ccc
-static int
9f5ccc
-event_slot_ref(struct event_slot_epoll *slot)
9f5ccc
-{
9f5ccc
-    if (!slot)
9f5ccc
-        return -1;
9f5ccc
-
9f5ccc
-    return GF_ATOMIC_INC(slot->ref);
9f5ccc
-}
9f5ccc
-
9f5ccc
 static struct event_slot_epoll *
9f5ccc
 event_slot_get(struct event_pool *event_pool, int idx)
9f5ccc
 {
9f5ccc
@@ -379,20 +394,13 @@ event_register_epoll(struct event_pool *event_pool, int fd,
9f5ccc
     if (destroy == 1)
9f5ccc
         goto out;
9f5ccc
 
9f5ccc
-    idx = event_slot_alloc(event_pool, fd, notify_poller_death);
9f5ccc
+    idx = event_slot_alloc(event_pool, fd, notify_poller_death, &slot);
9f5ccc
     if (idx == -1) {
9f5ccc
         gf_msg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND,
9f5ccc
                "could not find slot for fd=%d", fd);
9f5ccc
         return -1;
9f5ccc
     }
9f5ccc
 
9f5ccc
-    slot = event_slot_get(event_pool, idx);
9f5ccc
-    if (!slot) {
9f5ccc
-        gf_msg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND,
9f5ccc
-               "could not find slot for fd=%d idx=%d", fd, idx);
9f5ccc
-        return -1;
9f5ccc
-    }
9f5ccc
-
9f5ccc
     assert(slot->fd == fd);
9f5ccc
 
9f5ccc
     LOCK(&slot->lock);
9f5ccc
-- 
9f5ccc
1.8.3.1
9f5ccc