From a716a8ddd3df615009bcff3bd96dd9ae64cb5f68 Mon Sep 17 00:00:00 2001

From: Klaus Wenninger <klaus.wenninger@aon.at>

Date: Tue, 19 Mar 2019 21:36:15 +0100

Subject: [PATCH] Fix: sbd-pacemaker: make handling of cib-connection loss more

 robust

Exit pcmk-servant on graceful pacemaker shutdown and go back

to state before pacemaker was detected initially.

Purge all cib-traces otherwise and try to reconnect within timeout.

---

 src/sbd-inquisitor.c | 24 ++++++++++++++++++++----

 src/sbd-md.c         | 30 +++++++++++++++---------------

 src/sbd-pacemaker.c  | 38 +++++++++++++++++++++++++++++---------

 src/sbd.h            | 11 +++++++----

 4 files changed, 71 insertions(+), 32 deletions(-)

diff --git a/src/sbd-inquisitor.c b/src/sbd-inquisitor.c

index 9be6c99..77c6e4f 100644

--- a/src/sbd-inquisitor.c

+++ b/src/sbd-inquisitor.c

@@ -490,19 +490,19 @@ void inquisitor_child(void)

 					if (sbd_is_disk(s)) {

 						if (WIFEXITED(status)) {

 							switch(WEXITSTATUS(status)) {

-								case EXIT_MD_IO_FAIL:

+								case EXIT_MD_SERVANT_IO_FAIL:

 									DBGLOG(LOG_INFO, "Servant for %s requests to be disowned",

 										s->devname);

 									break;

-								case EXIT_MD_REQUEST_RESET:

+								case EXIT_MD_SERVANT_REQUEST_RESET:

 									cl_log(LOG_WARNING, "%s requested a reset", s->devname);

 									do_reset();

 									break;

-								case EXIT_MD_REQUEST_SHUTOFF:

+								case EXIT_MD_SERVANT_REQUEST_SHUTOFF:

 									cl_log(LOG_WARNING, "%s requested a shutoff", s->devname);

 									do_off();

 									break;

-								case EXIT_MD_REQUEST_CRASHDUMP:

+								case EXIT_MD_SERVANT_REQUEST_CRASHDUMP:

 									cl_log(LOG_WARNING, "%s requested a crashdump", s->devname);

 									do_crashdump();

 									break;

@@ -510,6 +510,22 @@ void inquisitor_child(void)

 									break;

 							}

 						}

+					} else if (sbd_is_pcmk(s)) {

+						if (WIFEXITED(status)) {

+							switch(WEXITSTATUS(status)) {

+								case EXIT_PCMK_SERVANT_GRACEFUL_SHUTDOWN:

+									DBGLOG(LOG_INFO, "PCMK-Servant has exited gracefully");

+									/* revert to state prior to pacemaker-detection */

+									s->restarts = 0;

+									s->restart_blocked = 0;

+									cluster_appeared = 0;

+									s->outdated = 1;

+									s->t_last.tv_sec = 0;

+									break;

+								default:

+									break;

+							}

+						}

 					}

 					cleanup_servant_by_pid(pid);

 				}

diff --git a/src/sbd-md.c b/src/sbd-md.c

index ba2c34d..c51d381 100644

--- a/src/sbd-md.c

+++ b/src/sbd-md.c

@@ -1061,19 +1061,19 @@ int servant_md(const char *diskname, int mode, const void* argp)

 	st = open_device(diskname, LOG_WARNING);

 	if (!st) {

-		exit(EXIT_MD_IO_FAIL);

+		exit(EXIT_MD_SERVANT_IO_FAIL);

 	}

 	s_header = header_get(st);

 	if (!s_header) {

 		cl_log(LOG_ERR, "Not a valid header on %s", diskname);

-		exit(EXIT_MD_IO_FAIL);

+		exit(EXIT_MD_SERVANT_IO_FAIL);

 	}

 	if (servant_check_timeout_inconsistent(s_header) < 0) {

 		cl_log(LOG_ERR, "Timeouts on %s do not match first device",

 				diskname);

-		exit(EXIT_MD_IO_FAIL);

+		exit(EXIT_MD_SERVANT_IO_FAIL);

 	}

 	if (s_header->minor_version > 0) {

@@ -1086,14 +1086,14 @@ int servant_md(const char *diskname, int mode, const void* argp)

 		cl_log(LOG_ERR,

 		       "No slot allocated, and automatic allocation failed for disk %s.",

 		       diskname);

-		rc = EXIT_MD_IO_FAIL;

+		rc = EXIT_MD_SERVANT_IO_FAIL;

 		goto out;

 	}

 	s_node = sector_alloc();

 	if (slot_read(st, mbox, s_node) < 0) {

 		cl_log(LOG_ERR, "Unable to read node entry on %s",

 				diskname);

-		exit(EXIT_MD_IO_FAIL);

+		exit(EXIT_MD_SERVANT_IO_FAIL);

 	}

 	cl_log(LOG_NOTICE, "Monitoring slot %d on disk %s", mbox, diskname);

@@ -1109,7 +1109,7 @@ int servant_md(const char *diskname, int mode, const void* argp)

 		if (mode > 0) {

 			if (mbox_read(st, mbox, s_mbox) < 0) {

 				cl_log(LOG_ERR, "mbox read failed during start-up in servant.");

-				rc = EXIT_MD_IO_FAIL;

+				rc = EXIT_MD_SERVANT_IO_FAIL;

 				goto out;

 			}

 			if (s_mbox->cmd != SBD_MSG_EXIT &&

@@ -1125,7 +1125,7 @@ int servant_md(const char *diskname, int mode, const void* argp)

 		DBGLOG(LOG_INFO, "First servant start - zeroing inbox");

 		memset(s_mbox, 0, sizeof(*s_mbox));

 		if (mbox_write(st, mbox, s_mbox) < 0) {

-			rc = EXIT_MD_IO_FAIL;

+			rc = EXIT_MD_SERVANT_IO_FAIL;

 			goto out;

 		}

 	}

@@ -1154,28 +1154,28 @@ int servant_md(const char *diskname, int mode, const void* argp)

 		s_header_retry = header_get(st);

 		if (!s_header_retry) {

 			cl_log(LOG_ERR, "No longer found a valid header on %s", diskname);

-			exit(EXIT_MD_IO_FAIL);

+			exit(EXIT_MD_SERVANT_IO_FAIL);

 		}

 		if (memcmp(s_header, s_header_retry, sizeof(*s_header)) != 0) {

 			cl_log(LOG_ERR, "Header on %s changed since start-up!", diskname);

-			exit(EXIT_MD_IO_FAIL);

+			exit(EXIT_MD_SERVANT_IO_FAIL);

 		}

 		free(s_header_retry);

 		s_node_retry = sector_alloc();

 		if (slot_read(st, mbox, s_node_retry) < 0) {

 			cl_log(LOG_ERR, "slot read failed in servant.");

-			exit(EXIT_MD_IO_FAIL);

+			exit(EXIT_MD_SERVANT_IO_FAIL);

 		}

 		if (memcmp(s_node, s_node_retry, sizeof(*s_node)) != 0) {

 			cl_log(LOG_ERR, "Node entry on %s changed since start-up!", diskname);

-			exit(EXIT_MD_IO_FAIL);

+			exit(EXIT_MD_SERVANT_IO_FAIL);

 		}

 		free(s_node_retry);

 		if (mbox_read(st, mbox, s_mbox) < 0) {

 			cl_log(LOG_ERR, "mbox read failed in servant.");

-			exit(EXIT_MD_IO_FAIL);

+			exit(EXIT_MD_SERVANT_IO_FAIL);

 		}

 		if (s_mbox->cmd > 0) {

@@ -1190,14 +1190,14 @@ int servant_md(const char *diskname, int mode, const void* argp)

 				sigqueue(ppid, SIG_TEST, signal_value);

 				break;

 			case SBD_MSG_RESET:

-				exit(EXIT_MD_REQUEST_RESET);

+				exit(EXIT_MD_SERVANT_REQUEST_RESET);

 			case SBD_MSG_OFF:

-				exit(EXIT_MD_REQUEST_SHUTOFF);

+				exit(EXIT_MD_SERVANT_REQUEST_SHUTOFF);

 			case SBD_MSG_EXIT:

 				sigqueue(ppid, SIG_EXITREQ, signal_value);

 				break;

 			case SBD_MSG_CRASHDUMP:

-				exit(EXIT_MD_REQUEST_CRASHDUMP);

+				exit(EXIT_MD_SERVANT_REQUEST_CRASHDUMP);

 			default:

 				/* FIXME:

 				   An "unknown" message might result

diff --git a/src/sbd-pacemaker.c b/src/sbd-pacemaker.c

index aac355a..c69fc55 100644

--- a/src/sbd-pacemaker.c

+++ b/src/sbd-pacemaker.c

@@ -103,6 +103,9 @@ static pe_working_set_t *data_set = NULL;

 static long last_refresh = 0;

+static int pcmk_clean_shutdown = 0;

+static int pcmk_shutdown = 0;

+

 static gboolean

 mon_timer_reconnect(gpointer data)

 {

@@ -128,10 +131,26 @@ mon_cib_connection_destroy(gpointer user_data)

 {

 	if (cib) {

 		cib->cmds->signoff(cib);

+		/* retrigger as last one might have been skipped */

+		mon_refresh_state(NULL);

+		if (pcmk_clean_shutdown) {

+			/* assume a graceful pacemaker-shutdown */

+			clean_up(EXIT_PCMK_SERVANT_GRACEFUL_SHUTDOWN);

+		}

+		/* getting here we aren't sure about the pacemaker-state

+		   so try to use the timeout to reconnect and get

+		   everything sorted out again

+		 */

+		pcmk_shutdown = 0;

 		set_servant_health(pcmk_health_transient, LOG_WARNING, "Disconnected from CIB");

 		timer_id_reconnect = g_timeout_add(reconnect_msec, mon_timer_reconnect, NULL);

 	}

 	cib_connected = 0;

+	/* no sense in looking into outdated cib, trying to apply patch, ... */

+	if (current_cib) {

+		free_xml(current_cib);

+		current_cib = NULL;

+	}

 	return;

 }

@@ -171,7 +190,7 @@ static gboolean

 mon_timer_notify(gpointer data)

 {

 	static int counter = 0;

-	int counter_max = timeout_watchdog / timeout_loop;

+	int counter_max = timeout_watchdog / timeout_loop / 2;

 	if (timer_id_notify > 0) {

 		g_source_remove(timer_id_notify);

@@ -280,11 +299,6 @@ compute_status(pe_working_set_t * data_set)

     } else if (node->details->pending) {

         set_servant_health(pcmk_health_pending, LOG_WARNING, "Node state: pending");

-#if 0

-    } else if (node->details->shutdown) {

-        set_servant_health(pcmk_health_shutdown, LOG_WARNING, "Node state: shutting down");

-#endif

-

     } else if (data_set->flags & pe_flag_have_quorum) {

         set_servant_health(pcmk_health_online, LOG_INFO, "Node state: online");

         ever_had_quorum = TRUE;

@@ -315,6 +329,12 @@ compute_status(pe_working_set_t * data_set)

         }

     }

+    if (node->details->shutdown) {

+        pcmk_shutdown = 1;

+    }

+    if (pcmk_shutdown && !(node->details->running_rsc)) {

+        pcmk_clean_shutdown = 1;

+    }

     notify_parent();

     return;

 }

@@ -339,7 +359,7 @@ crm_diff_update(const char *event, xmlNode * msg)

         static mainloop_timer_t *refresh_timer = NULL;

         if(refresh_timer == NULL) {

-            refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL);

+            refresh_timer = mainloop_timer_add("refresh", reconnect_msec, FALSE, mon_trigger_refresh, NULL);

             refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_state, refresh_timer);

         }

@@ -369,9 +389,9 @@ crm_diff_update(const char *event, xmlNode * msg)

 	}

     /* Refresh

-     * - immediately if the last update was more than 5s ago

+     * - immediately if the last update was more than 1s ago

      * - every 10 updates

-     * - at most 2s after the last update

+     * - at most 1s after the last update

      */

     if (updates > 10 || (now - last_refresh) > (reconnect_msec / 1000)) {

         mon_refresh_state(refresh_timer);

diff --git a/src/sbd.h b/src/sbd.h

index 6fe07f9..3b05a11 100644

--- a/src/sbd.h

+++ b/src/sbd.h

@@ -54,10 +54,13 @@

 /* FIXME: should add dynamic check of SIG_XX >= SIGRTMAX */

 /* exit status for disk-servant */

-#define EXIT_MD_IO_FAIL             20

-#define EXIT_MD_REQUEST_RESET       21

-#define EXIT_MD_REQUEST_SHUTOFF     22

-#define EXIT_MD_REQUEST_CRASHDUMP   23

+#define EXIT_MD_SERVANT_IO_FAIL             20

+#define EXIT_MD_SERVANT_REQUEST_RESET       21

+#define EXIT_MD_SERVANT_REQUEST_SHUTOFF     22

+#define EXIT_MD_SERVANT_REQUEST_CRASHDUMP   23

+

+/* exit status for pcmk-servant */

+#define EXIT_PCMK_SERVANT_GRACEFUL_SHUTDOWN 30

 #define HOG_CHAR	0xff

 #define SECTOR_NAME_MAX 63

-- 

1.8.3.1