From f199094cb61341a47c98a8ed91b293446182b5a9 Mon Sep 17 00:00:00 2001 From: Mohit Agrawal Date: Thu, 3 Oct 2019 14:06:52 +0530 Subject: [PATCH 333/335] rpc: Synchronize slot allocation code Problem: Current slot allocation/deallocation code path is not synchronized.There are scenario when due to race condition in slot allocation/deallocation code path brick is crashed. Solution: Synchronize slot allocation/deallocation code path to avoid the issue > Change-Id: I4fb659a75234218ffa0e5e0bf9308f669f75fc25 > Fixes: bz#1763036 > Signed-off-by: Mohit Agrawal > (Reviewed on upstream link https://review.gluster.org/#/c/glusterfs/+/23508/) > (Cherry pick from commit faf5ac13c4ee00a05e9451bf8da3be2a9043bbf2) Change-Id: I4fb659a75234218ffa0e5e0bf9308f669f75fc25 BUG: 1741193 Signed-off-by: Mohit Agrawal Reviewed-on: https://code.engineering.redhat.com/gerrit/185827 Tested-by: RHGS Build Bot Reviewed-by: Sunil Kumar Heggodu Gopala Acharya --- libglusterfs/src/event-epoll.c | 74 +++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/libglusterfs/src/event-epoll.c b/libglusterfs/src/event-epoll.c index 0cec47e..65f5efd 100644 --- a/libglusterfs/src/event-epoll.c +++ b/libglusterfs/src/event-epoll.c @@ -69,15 +69,27 @@ __event_newtable(struct event_pool *event_pool, int table_idx) } static int +event_slot_ref(struct event_slot_epoll *slot) +{ + if (!slot) + return -1; + + return GF_ATOMIC_INC(slot->ref); +} + +static int __event_slot_alloc(struct event_pool *event_pool, int fd, - char notify_poller_death) + char notify_poller_death, struct event_slot_epoll **slot) { int i = 0; + int j = 0; int table_idx = -1; int gen = -1; struct event_slot_epoll *table = NULL; - for (i = 0; i < EVENT_EPOLL_TABLES; i++) { +retry: + + while (i < EVENT_EPOLL_TABLES) { switch (event_pool->slots_used[i]) { case EVENT_EPOLL_SLOTS: continue; @@ -98,6 +110,7 @@ __event_slot_alloc(struct event_pool *event_pool, int fd, if (table) /* break out of the loop */ break; + i++; } if (!table) @@ -105,20 +118,20 @@ __event_slot_alloc(struct event_pool *event_pool, int fd, table_idx = i; - for (i = 0; i < EVENT_EPOLL_SLOTS; i++) { - if (table[i].fd == -1) { + for (j = 0; j < EVENT_EPOLL_SLOTS; j++) { + if (table[j].fd == -1) { /* wipe everything except bump the generation */ - gen = table[i].gen; - memset(&table[i], 0, sizeof(table[i])); - table[i].gen = gen + 1; + gen = table[j].gen; + memset(&table[j], 0, sizeof(table[j])); + table[j].gen = gen + 1; - LOCK_INIT(&table[i].lock); - INIT_LIST_HEAD(&table[i].poller_death); + LOCK_INIT(&table[j].lock); + INIT_LIST_HEAD(&table[j].poller_death); - table[i].fd = fd; + table[j].fd = fd; if (notify_poller_death) { - table[i].idx = table_idx * EVENT_EPOLL_SLOTS + i; - list_add_tail(&table[i].poller_death, + table[j].idx = table_idx * EVENT_EPOLL_SLOTS + j; + list_add_tail(&table[j].poller_death, &event_pool->poller_death); } @@ -128,18 +141,26 @@ __event_slot_alloc(struct event_pool *event_pool, int fd, } } - return table_idx * EVENT_EPOLL_SLOTS + i; + if (j == EVENT_EPOLL_SLOTS) { + table = NULL; + i++; + goto retry; + } else { + (*slot) = &table[j]; + event_slot_ref(*slot); + return table_idx * EVENT_EPOLL_SLOTS + j; + } } static int event_slot_alloc(struct event_pool *event_pool, int fd, - char notify_poller_death) + char notify_poller_death, struct event_slot_epoll **slot) { int idx = -1; pthread_mutex_lock(&event_pool->mutex); { - idx = __event_slot_alloc(event_pool, fd, notify_poller_death); + idx = __event_slot_alloc(event_pool, fd, notify_poller_death, slot); } pthread_mutex_unlock(&event_pool->mutex); @@ -153,6 +174,7 @@ __event_slot_dealloc(struct event_pool *event_pool, int idx) int offset = 0; struct event_slot_epoll *table = NULL; struct event_slot_epoll *slot = NULL; + int fd = -1; table_idx = idx / EVENT_EPOLL_SLOTS; offset = idx % EVENT_EPOLL_SLOTS; @@ -164,11 +186,13 @@ __event_slot_dealloc(struct event_pool *event_pool, int idx) slot = &table[offset]; slot->gen++; + fd = slot->fd; slot->fd = -1; slot->handled_error = 0; slot->in_handler = 0; list_del_init(&slot->poller_death); - event_pool->slots_used[table_idx]--; + if (fd != -1) + event_pool->slots_used[table_idx]--; return; } @@ -185,15 +209,6 @@ event_slot_dealloc(struct event_pool *event_pool, int idx) return; } -static int -event_slot_ref(struct event_slot_epoll *slot) -{ - if (!slot) - return -1; - - return GF_ATOMIC_INC(slot->ref); -} - static struct event_slot_epoll * event_slot_get(struct event_pool *event_pool, int idx) { @@ -379,20 +394,13 @@ event_register_epoll(struct event_pool *event_pool, int fd, if (destroy == 1) goto out; - idx = event_slot_alloc(event_pool, fd, notify_poller_death); + idx = event_slot_alloc(event_pool, fd, notify_poller_death, &slot); if (idx == -1) { gf_msg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, "could not find slot for fd=%d", fd); return -1; } - slot = event_slot_get(event_pool, idx); - if (!slot) { - gf_msg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, - "could not find slot for fd=%d idx=%d", fd, idx); - return -1; - } - assert(slot->fd == fd); LOCK(&slot->lock); -- 1.8.3.1