commit e360e34b1fd2cd69c3a08bfb7fa4fe75281b942a

Author: Tomas Korbar <tkorbar@redhat.com>

Date:   Tue May 19 08:42:12 2020 +0200

    restart: fix rare segfault on shutdown

diff --git a/memcached.c b/memcached.c

index d769b4a..ac03b93 100644

--- a/memcached.c

+++ b/memcached.c

@@ -1009,6 +1009,18 @@ static void conn_shrink(conn *c) {

     }

 }

+// Since some connections might be off on side threads and some are managed as

+// listeners we need to walk through them all from a central point.

+// Must be called with all worker threads hung or in the process of closing.

+void conn_close_all(void) {

+    int i;

+    for (i = 0; i < max_fds; i++) {

+        if (conns[i] && conns[i]->state != conn_closed) {

+            conn_close(conns[i]);

+        }

+    }

+}

+

 /**

  * Convert a state name to a human readable form.

  */

@@ -9860,13 +9872,6 @@ int main (int argc, char **argv) {

     }

     stop_threads();

-    int i;

-    // FIXME: make a function callable from threads.c

-    for (i = 0; i < max_fds; i++) {

-        if (conns[i] && conns[i]->state != conn_closed) {

-            conn_close(conns[i]);

-        }

-    }

     if (memory_file != NULL && stop_main_loop == GRACE_STOP) {

         restart_mmap_close();

     }

diff --git a/memcached.h b/memcached.h

index 6b1fe4a..bc2b395 100644

--- a/memcached.h

+++ b/memcached.h

@@ -814,9 +814,8 @@ enum delta_result_type add_delta(conn *c, const char *key,

                                  const int64_t delta, char *buf,

                                  uint64_t *cas);

 void accept_new_conns(const bool do_accept);

-conn *conn_from_freelist(void);

-bool  conn_add_to_freelist(conn *c);

 void  conn_close_idle(conn *c);

+void  conn_close_all(void);

 item *item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbytes);

 #define DO_UPDATE true

 #define DONT_UPDATE false

diff --git a/thread.c b/thread.c

index 7cba01e..6e19a2e 100644

--- a/thread.c

+++ b/thread.c

@@ -205,6 +205,7 @@ void stop_threads(void) {

     if (settings.verbose > 0)

         fprintf(stderr, "asking workers to stop\n");

     buf[0] = 's';

+    pthread_mutex_lock(&worker_hang_lock);

     pthread_mutex_lock(&init_lock);

     init_count = 0;

     for (i = 0; i < settings.num_threads; i++) {

@@ -216,6 +217,8 @@ void stop_threads(void) {

     wait_for_thread_registration(settings.num_threads);

     pthread_mutex_unlock(&init_lock);

+    // All of the workers are hung but haven't done cleanup yet.

+

     if (settings.verbose > 0)

         fprintf(stderr, "asking background threads to stop\n");

@@ -237,6 +240,17 @@ void stop_threads(void) {

     if (settings.verbose > 0)

         fprintf(stderr, "stopped idle timeout thread\n");

+    // Close all connections then let the workers finally exit.

+    if (settings.verbose > 0)

+        fprintf(stderr, "closing connections\n");

+    conn_close_all();

+    pthread_mutex_unlock(&worker_hang_lock);

+    if (settings.verbose > 0)

+        fprintf(stderr, "reaping worker threads\n");

+    for (i = 0; i < settings.num_threads; i++) {

+        pthread_join(threads[i].thread_id, NULL);

+    }

+

     if (settings.verbose > 0)

         fprintf(stderr, "all background threads stopped\n");