diff --git a/SOURCES/042-unfencing-loop.patch b/SOURCES/042-unfencing-loop.patch new file mode 100644 index 0000000..2bae706 --- /dev/null +++ b/SOURCES/042-unfencing-loop.patch @@ -0,0 +1,733 @@ +From 6dcd6b51d7d3993bc483588d6ed75077518ed600 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 4 Jun 2021 16:30:55 -0500 +Subject: [PATCH 01/11] Low: controller: check whether unfenced node was remote + node + +... so the controller can indicate the node is remote (if known at that point, +which is not guaranteed) when setting unfencing-related node attributes. +--- + daemons/controld/controld_fencing.c | 21 ++++++++++++++++++--- + 1 file changed, 18 insertions(+), 3 deletions(-) + +diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c +index 23dff28..0fba661 100644 +--- a/daemons/controld/controld_fencing.c ++++ b/daemons/controld/controld_fencing.c +@@ -757,15 +757,30 @@ tengine_stonith_callback(stonith_t *stonith, stonith_callback_data_t *data) + if (pcmk__str_eq("on", op, pcmk__str_casei)) { + const char *value = NULL; + char *now = crm_ttoa(time(NULL)); ++ gboolean is_remote_node = FALSE; ++ ++ /* This check is not 100% reliable, since this node is not ++ * guaranteed to have the remote node cached. However, it ++ * doesn't have to be reliable, since the attribute manager can ++ * learn a node's "remoteness" by other means sooner or later. ++ * This allows it to learn more quickly if this node does have ++ * the information. ++ */ ++ if (g_hash_table_lookup(crm_remote_peer_cache, uuid) != NULL) { ++ is_remote_node = TRUE; ++ } + +- update_attrd(target, CRM_ATTR_UNFENCED, now, NULL, FALSE); ++ update_attrd(target, CRM_ATTR_UNFENCED, now, NULL, ++ is_remote_node); + free(now); + + value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_ALL); +- update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL, FALSE); ++ update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL, ++ is_remote_node); + + value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_SECURE); +- update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL, FALSE); ++ update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL, ++ is_remote_node); + + } else if (action->sent_update == FALSE) { + send_stonith_update(action, target, uuid); +-- +1.8.3.1 + + +From 3ef6d9403f68ab8559c45cc99f5a8da05ca6420b Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 7 Jun 2021 10:50:36 -0500 +Subject: [PATCH 02/11] Refactor: pacemaker-attrd: functionize adding remote + node to cache + +... for future reuse +--- + daemons/attrd/attrd_commands.c | 34 +++++++++++++++++++++++----------- + 1 file changed, 23 insertions(+), 11 deletions(-) + +diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c +index 731c243..93a165b 100644 +--- a/daemons/attrd/attrd_commands.c ++++ b/daemons/attrd/attrd_commands.c +@@ -102,6 +102,28 @@ free_attribute(gpointer data) + } + } + ++/*! ++ * \internal ++ * \brief Ensure a Pacemaker Remote node is in the correct peer cache ++ * ++ * \param[in] ++ */ ++static void ++cache_remote_node(const char *node_name) ++{ ++ /* If we previously assumed this node was an unseen cluster node, ++ * remove its entry from the cluster peer cache. ++ */ ++ crm_node_t *dup = crm_find_peer(0, node_name); ++ ++ if (dup && (dup->uuid == NULL)) { ++ reap_crm_member(0, node_name); ++ } ++ ++ // Ensure node is in the remote peer cache ++ CRM_ASSERT(crm_remote_peer_get(node_name) != NULL); ++} ++ + static xmlNode * + build_attribute_xml( + xmlNode *parent, const char *name, const char *set, const char *uuid, unsigned int timeout_ms, const char *user, +@@ -709,17 +731,7 @@ attrd_lookup_or_create_value(GHashTable *values, const char *host, xmlNode *xml) + + crm_element_value_int(xml, PCMK__XA_ATTR_IS_REMOTE, &is_remote); + if (is_remote) { +- /* If we previously assumed this node was an unseen cluster node, +- * remove its entry from the cluster peer cache. +- */ +- crm_node_t *dup = crm_find_peer(0, host); +- +- if (dup && (dup->uuid == NULL)) { +- reap_crm_member(0, host); +- } +- +- /* Ensure this host is in the remote peer cache */ +- CRM_ASSERT(crm_remote_peer_get(host) != NULL); ++ cache_remote_node(host); + } + + if (v == NULL) { +-- +1.8.3.1 + + +From 6fac2c71bc2c56870ac828d7cd7b7c799279c47e Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 7 Jun 2021 10:39:34 -0500 +Subject: [PATCH 03/11] Refactor: pacemaker-attrd: don't try to remove votes + for remote nodes + +Remote nodes never vote. + +This has no effect in practice since the removal would simply do nothing, +but we might as well not waste time trying. +--- + daemons/attrd/attrd_commands.c | 11 ++++++----- + 1 file changed, 6 insertions(+), 5 deletions(-) + +diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c +index 93a165b..dbe777e 100644 +--- a/daemons/attrd/attrd_commands.c ++++ b/daemons/attrd/attrd_commands.c +@@ -976,7 +976,8 @@ attrd_election_cb(gpointer user_data) + void + attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *data) + { +- bool remove_voter = FALSE; ++ bool gone = false; ++ bool is_remote = pcmk_is_set(peer->flags, crm_remote_node); + + switch (kind) { + case crm_status_uname: +@@ -984,7 +985,7 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da + + case crm_status_processes: + if (!pcmk_is_set(peer->processes, crm_get_cluster_proc())) { +- remove_voter = TRUE; ++ gone = true; + } + break; + +@@ -1000,13 +1001,13 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da + } else { + // Remove all attribute values associated with lost nodes + attrd_peer_remove(peer->uname, FALSE, "loss"); +- remove_voter = TRUE; ++ gone = true; + } + break; + } + +- // In case an election is in progress, remove any vote by the node +- if (remove_voter) { ++ // Remove votes from cluster nodes that leave, in case election in progress ++ if (gone && !is_remote) { + attrd_remove_voter(peer); + } + } +-- +1.8.3.1 + + +From 54089fc663d6aaf10ca164c6c94b3b17237788de Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 7 Jun 2021 10:40:06 -0500 +Subject: [PATCH 04/11] Low: pacemaker-attrd: check for remote nodes in peer + update callback + +If a remote node was started before the local cluster node joined the cluster, +the cluster node will assume its node attributes are for a cluster node until +it learns otherwise. Check for remoteness in the peer update callback, to have +another way we can learn it. +--- + daemons/attrd/attrd_commands.c | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c +index dbe777e..5f6a754 100644 +--- a/daemons/attrd/attrd_commands.c ++++ b/daemons/attrd/attrd_commands.c +@@ -1009,6 +1009,10 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da + // Remove votes from cluster nodes that leave, in case election in progress + if (gone && !is_remote) { + attrd_remove_voter(peer); ++ ++ // Ensure remote nodes that come up are in the remote node cache ++ } else if (!gone && is_remote) { ++ cache_remote_node(peer->uname); + } + } + +-- +1.8.3.1 + + +From 8c048df0312d0d9c857d87b570a352429a710928 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 7 Jun 2021 11:29:12 -0500 +Subject: [PATCH 05/11] Log: pacemaker-attrd: log peer status changes + +--- + daemons/attrd/attrd_commands.c | 9 +++++++++ + 1 file changed, 9 insertions(+) + +diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c +index 5f6a754..d6d179b 100644 +--- a/daemons/attrd/attrd_commands.c ++++ b/daemons/attrd/attrd_commands.c +@@ -972,6 +972,7 @@ attrd_election_cb(gpointer user_data) + return FALSE; + } + ++#define state_text(state) ((state)? (const char *)(state) : "in unknown state") + + void + attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *data) +@@ -981,15 +982,23 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da + + switch (kind) { + case crm_status_uname: ++ crm_debug("%s node %s is now %s", ++ (is_remote? "Remote" : "Cluster"), ++ peer->uname, state_text(peer->state)); + break; + + case crm_status_processes: + if (!pcmk_is_set(peer->processes, crm_get_cluster_proc())) { + gone = true; + } ++ crm_debug("Node %s is %s a peer", ++ peer->uname, (gone? "no longer" : "now")); + break; + + case crm_status_nstate: ++ crm_debug("%s node %s is now %s (was %s)", ++ (is_remote? "Remote" : "Cluster"), ++ peer->uname, state_text(peer->state), state_text(data)); + if (pcmk__str_eq(peer->state, CRM_NODE_MEMBER, pcmk__str_casei)) { + /* If we're the writer, send new peers a list of all attributes + * (unless it's a remote node, which doesn't run its own attrd) +-- +1.8.3.1 + + +From 1dcc8dee4990cf0dbdec0e14db6d9a3ad67a41d5 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 7 Jun 2021 11:13:53 -0500 +Subject: [PATCH 06/11] Low: pacemaker-attrd: ensure node ID is only set for + attributes when known + +In most cases, attribute updates contained the node ID, and the node ID was +used by other code, only if known (i.e. positive). However a couple places did +not check this, so add that. + +I am unsure whether the missing check caused problems in practice, but there +appears to be the possibility that a remote node would wrongly be added to the +cluster node cache. +--- + daemons/attrd/attrd_commands.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c +index d6d179b..b3f441c 100644 +--- a/daemons/attrd/attrd_commands.c ++++ b/daemons/attrd/attrd_commands.c +@@ -136,7 +136,9 @@ build_attribute_xml( + crm_xml_add(xml, PCMK__XA_ATTR_UUID, uuid); + crm_xml_add(xml, PCMK__XA_ATTR_USER, user); + crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, peer); +- crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, peerid); ++ if (peerid > 0) { ++ crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, peerid); ++ } + crm_xml_add(xml, PCMK__XA_ATTR_VALUE, value); + crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, timeout_ms/1000); + crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, is_private); +@@ -937,7 +939,7 @@ attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host, bool filter) + /* If this is a cluster node whose node ID we are learning, remember it */ + if ((v->nodeid == 0) && (v->is_remote == FALSE) + && (crm_element_value_int(xml, PCMK__XA_ATTR_NODE_ID, +- (int*)&v->nodeid) == 0)) { ++ (int*)&v->nodeid) == 0) && (v->nodeid > 0)) { + + crm_node_t *known_peer = crm_get_peer(v->nodeid, host); + +-- +1.8.3.1 + + +From 8d12490e88b558d01db37a38f7d35175c6d2d69a Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 10 Jun 2021 17:25:57 -0500 +Subject: [PATCH 07/11] Refactor: pacemaker-attrd: functionize processing a + sync response + +... for code isolation, and because we need to add more to it +--- + daemons/attrd/attrd_commands.c | 59 ++++++++++++++++++++++++++++-------------- + 1 file changed, 39 insertions(+), 20 deletions(-) + +diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c +index b3f441c..d02d3e6 100644 +--- a/daemons/attrd/attrd_commands.c ++++ b/daemons/attrd/attrd_commands.c +@@ -572,6 +572,43 @@ attrd_peer_clear_failure(crm_node_t *peer, xmlNode *xml) + } + + /*! ++ * \internal ++ * \brief Load attributes from a peer sync response ++ * ++ * \param[in] peer Peer that sent clear request ++ * \param[in] peer_won Whether peer is the attribute writer ++ * \param[in] xml Request XML ++ */ ++static void ++process_peer_sync_response(crm_node_t *peer, bool peer_won, xmlNode *xml) ++{ ++ crm_info("Processing " PCMK__ATTRD_CMD_SYNC_RESPONSE " from %s", ++ peer->uname); ++ ++ if (peer_won) { ++ /* Initialize the "seen" flag for all attributes to cleared, so we can ++ * detect attributes that local node has but the writer doesn't. ++ */ ++ clear_attribute_value_seen(); ++ } ++ ++ // Process each attribute update in the sync response ++ for (xmlNode *child = pcmk__xml_first_child(xml); child != NULL; ++ child = pcmk__xml_next(child)) { ++ attrd_peer_update(peer, child, ++ crm_element_value(child, PCMK__XA_ATTR_NODE_NAME), ++ TRUE); ++ } ++ ++ if (peer_won) { ++ /* If any attributes are still not marked as seen, the writer doesn't ++ * know about them, so send all peers an update with them. ++ */ ++ attrd_current_only_attribute_update(peer, xml); ++ } ++} ++ ++/*! + \internal + \brief Broadcast private attribute for local node with protocol version + */ +@@ -596,7 +633,7 @@ attrd_peer_message(crm_node_t *peer, xmlNode *xml) + const char *op = crm_element_value(xml, PCMK__XA_TASK); + const char *election_op = crm_element_value(xml, F_CRM_TASK); + const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME); +- bool peer_won = FALSE; ++ bool peer_won = false; + + if (election_op) { + attrd_handle_election_op(peer, xml); +@@ -631,25 +668,7 @@ attrd_peer_message(crm_node_t *peer, xmlNode *xml) + + } else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_SYNC_RESPONSE, pcmk__str_casei) + && !pcmk__str_eq(peer->uname, attrd_cluster->uname, pcmk__str_casei)) { +- xmlNode *child = NULL; +- +- crm_info("Processing %s from %s", op, peer->uname); +- +- /* Clear the seen flag for attribute processing held only in the own node. */ +- if (peer_won) { +- clear_attribute_value_seen(); +- } +- +- for (child = pcmk__xml_first_child(xml); child != NULL; +- child = pcmk__xml_next(child)) { +- host = crm_element_value(child, PCMK__XA_ATTR_NODE_NAME); +- attrd_peer_update(peer, child, host, TRUE); +- } +- +- if (peer_won) { +- /* Synchronize if there is an attribute held only by own node that Writer does not have. */ +- attrd_current_only_attribute_update(peer, xml); +- } ++ process_peer_sync_response(peer, peer_won, xml); + } + } + +-- +1.8.3.1 + + +From a890a0e5bbbcabf907f51ed0460868035f72464d Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 11 Jun 2021 14:40:39 -0500 +Subject: [PATCH 08/11] Refactor: pacemaker-attrd: functionize broadcasting + local override + +... for code isolation +--- + daemons/attrd/attrd_commands.c | 42 +++++++++++++++++++++++++++++------------- + 1 file changed, 29 insertions(+), 13 deletions(-) + +diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c +index d02d3e6..4783427 100644 +--- a/daemons/attrd/attrd_commands.c ++++ b/daemons/attrd/attrd_commands.c +@@ -804,6 +804,34 @@ attrd_current_only_attribute_update(crm_node_t *peer, xmlNode *xml) + free_xml(sync); + } + ++/*! ++ * \internal ++ * \brief Override an attribute sync with a local value ++ * ++ * Broadcast the local node's value for an attribute that's different from the ++ * value provided in a peer's attribute synchronization response. This ensures a ++ * node's values for itself take precedence and all peers are kept in sync. ++ * ++ * \param[in] a Attribute entry to override ++ * ++ * \return Local instance of attribute value ++ */ ++static attribute_value_t * ++broadcast_local_value(attribute_t *a) ++{ ++ attribute_value_t *v = g_hash_table_lookup(a->values, attrd_cluster->uname); ++ xmlNode *sync = create_xml_node(NULL, __func__); ++ ++ crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE); ++ build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, ++ a->user, a->is_private, v->nodename, v->nodeid, ++ v->current, FALSE); ++ attrd_xml_add_writer(sync); ++ send_attrd_message(NULL, sync); ++ free_xml(sync); ++ return v; ++} ++ + void + attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host, bool filter) + { +@@ -899,21 +927,9 @@ attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host, bool filter) + if (filter && !pcmk__str_eq(v->current, value, pcmk__str_casei) + && pcmk__str_eq(host, attrd_cluster->uname, pcmk__str_casei)) { + +- xmlNode *sync = create_xml_node(NULL, __func__); +- + crm_notice("%s[%s]: local value '%s' takes priority over '%s' from %s", + attr, host, v->current, value, peer->uname); +- +- crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE); +- v = g_hash_table_lookup(a->values, host); +- build_attribute_xml(sync, attr, a->set, a->uuid, a->timeout_ms, a->user, +- a->is_private, v->nodename, v->nodeid, v->current, FALSE); +- +- attrd_xml_add_writer(sync); +- +- /* Broadcast in case any other nodes had the inconsistent value */ +- send_attrd_message(NULL, sync); +- free_xml(sync); ++ v = broadcast_local_value(a); + + } else if (!pcmk__str_eq(v->current, value, pcmk__str_casei)) { + crm_notice("Setting %s[%s]: %s -> %s " CRM_XS " from %s", +-- +1.8.3.1 + + +From f6f65e3dab070f1bbdf6d1383f4d6173a8840bc9 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 11 Jun 2021 14:50:29 -0500 +Subject: [PATCH 09/11] Log: pacemaker-attrd: improve messages when + broadcasting local-only values + +The traces aren't necessary since build_attribute_xml() already logs the same +info at debug. Also, rename function for clarity, and make static. +--- + daemons/attrd/attrd_commands.c | 35 ++++++++++++++++------------------- + 1 file changed, 16 insertions(+), 19 deletions(-) + +diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c +index 4783427..356defb 100644 +--- a/daemons/attrd/attrd_commands.c ++++ b/daemons/attrd/attrd_commands.c +@@ -51,11 +51,12 @@ GHashTable *attributes = NULL; + + void write_attribute(attribute_t *a, bool ignore_delay); + void write_or_elect_attribute(attribute_t *a); +-void attrd_current_only_attribute_update(crm_node_t *peer, xmlNode *xml); + void attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host, bool filter); + void attrd_peer_sync(crm_node_t *peer, xmlNode *xml); + void attrd_peer_remove(const char *host, gboolean uncache, const char *source); + ++static void broadcast_unseen_local_values(crm_node_t *peer, xmlNode *xml); ++ + static gboolean + send_attrd_message(crm_node_t * node, xmlNode * data) + { +@@ -604,7 +605,7 @@ process_peer_sync_response(crm_node_t *peer, bool peer_won, xmlNode *xml) + /* If any attributes are still not marked as seen, the writer doesn't + * know about them, so send all peers an update with them. + */ +- attrd_current_only_attribute_update(peer, xml); ++ broadcast_unseen_local_values(peer, xml); + } + } + +@@ -768,40 +769,36 @@ attrd_lookup_or_create_value(GHashTable *values, const char *host, xmlNode *xml) + return(v); + } + +-void +-attrd_current_only_attribute_update(crm_node_t *peer, xmlNode *xml) ++void ++broadcast_unseen_local_values(crm_node_t *peer, xmlNode *xml) + { + GHashTableIter aIter; + GHashTableIter vIter; +- attribute_t *a; ++ attribute_t *a = NULL; + attribute_value_t *v = NULL; +- xmlNode *sync = create_xml_node(NULL, __func__); +- gboolean build = FALSE; +- +- crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE); ++ xmlNode *sync = NULL; + + g_hash_table_iter_init(&aIter, attributes); + while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) { + g_hash_table_iter_init(&vIter, a->values); + while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) { +- if (pcmk__str_eq(v->nodename, attrd_cluster->uname, pcmk__str_casei) && v->seen == FALSE) { +- crm_trace("Syncing %s[%s] = %s to everyone.(from local only attributes)", a->id, v->nodename, v->current); +- +- build = TRUE; ++ if (!(v->seen) && pcmk__str_eq(v->nodename, attrd_cluster->uname, ++ pcmk__str_casei)) { ++ if (sync == NULL) { ++ sync = create_xml_node(NULL, __func__); ++ crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE); ++ } + build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, a->is_private, + v->nodename, v->nodeid, v->current, (a->timeout_ms && a->timer ? TRUE : FALSE)); +- } else { +- crm_trace("Local attribute(%s[%s] = %s) was ignore.(another host) : [%s]", a->id, v->nodename, v->current, attrd_cluster->uname); +- continue; + } + } + } + +- if (build) { +- crm_debug("Syncing values to everyone.(from local only attributes)"); ++ if (sync != NULL) { ++ crm_debug("Broadcasting local-only values"); + send_attrd_message(NULL, sync); ++ free_xml(sync); + } +- free_xml(sync); + } + + /*! +-- +1.8.3.1 + + +From ab90ffb785ea018556f216b8f540f8c3429a3947 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 11 Jun 2021 15:04:20 -0500 +Subject: [PATCH 10/11] Refactor: pacemaker-attrd: simplify attribute XML + creation function + +... and rename for clarity +--- + daemons/attrd/attrd_commands.c | 48 ++++++++++++++++++++++++------------------ + 1 file changed, 27 insertions(+), 21 deletions(-) + +diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c +index 356defb..5b32a77 100644 +--- a/daemons/attrd/attrd_commands.c ++++ b/daemons/attrd/attrd_commands.c +@@ -125,25 +125,35 @@ cache_remote_node(const char *node_name) + CRM_ASSERT(crm_remote_peer_get(node_name) != NULL); + } + ++/*! ++ * \internal ++ * \brief Create an XML representation of an attribute for use in peer messages ++ * ++ * \param[in] parent Create attribute XML as child element of this element ++ * \param[in] a Attribute to represent ++ * \param[in] v Attribute value to represent ++ * \param[in] force_write If true, value should be written even if unchanged ++ * ++ * \return XML representation of attribute ++ */ + static xmlNode * +-build_attribute_xml( +- xmlNode *parent, const char *name, const char *set, const char *uuid, unsigned int timeout_ms, const char *user, +- gboolean is_private, const char *peer, uint32_t peerid, const char *value, gboolean is_force_write) ++add_attribute_value_xml(xmlNode *parent, attribute_t *a, attribute_value_t *v, ++ bool force_write) + { + xmlNode *xml = create_xml_node(parent, __func__); + +- crm_xml_add(xml, PCMK__XA_ATTR_NAME, name); +- crm_xml_add(xml, PCMK__XA_ATTR_SET, set); +- crm_xml_add(xml, PCMK__XA_ATTR_UUID, uuid); +- crm_xml_add(xml, PCMK__XA_ATTR_USER, user); +- crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, peer); +- if (peerid > 0) { +- crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, peerid); ++ crm_xml_add(xml, PCMK__XA_ATTR_NAME, a->id); ++ crm_xml_add(xml, PCMK__XA_ATTR_SET, a->set); ++ crm_xml_add(xml, PCMK__XA_ATTR_UUID, a->uuid); ++ crm_xml_add(xml, PCMK__XA_ATTR_USER, a->user); ++ crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, v->nodename); ++ if (v->nodeid > 0) { ++ crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, v->nodeid); + } +- crm_xml_add(xml, PCMK__XA_ATTR_VALUE, value); +- crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, timeout_ms/1000); +- crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, is_private); +- crm_xml_add_int(xml, PCMK__XA_ATTR_FORCE, is_force_write); ++ crm_xml_add(xml, PCMK__XA_ATTR_VALUE, v->current); ++ crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, a->timeout_ms / 1000); ++ crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, a->is_private); ++ crm_xml_add_int(xml, PCMK__XA_ATTR_FORCE, force_write); + + return xml; + } +@@ -695,8 +705,7 @@ attrd_peer_sync(crm_node_t *peer, xmlNode *xml) + g_hash_table_iter_init(&vIter, a->values); + while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) { + crm_debug("Syncing %s[%s] = %s to %s", a->id, v->nodename, v->current, peer?peer->uname:"everyone"); +- build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, a->is_private, +- v->nodename, v->nodeid, v->current, FALSE); ++ add_attribute_value_xml(sync, a, v, false); + } + } + +@@ -788,8 +797,7 @@ broadcast_unseen_local_values(crm_node_t *peer, xmlNode *xml) + sync = create_xml_node(NULL, __func__); + crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE); + } +- build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, a->is_private, +- v->nodename, v->nodeid, v->current, (a->timeout_ms && a->timer ? TRUE : FALSE)); ++ add_attribute_value_xml(sync, a, v, a->timeout_ms && a->timer); + } + } + } +@@ -820,9 +828,7 @@ broadcast_local_value(attribute_t *a) + xmlNode *sync = create_xml_node(NULL, __func__); + + crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE); +- build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, +- a->user, a->is_private, v->nodename, v->nodeid, +- v->current, FALSE); ++ add_attribute_value_xml(sync, a, v, false); + attrd_xml_add_writer(sync); + send_attrd_message(NULL, sync); + free_xml(sync); +-- +1.8.3.1 + + +From 540d74130c5c8d9c626d6c50475e4dc4f64234e7 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 4 Jun 2021 16:34:26 -0500 +Subject: [PATCH 11/11] Fix: pacemaker-attrd: avoid repeated unfencing of + remote nodes + +The attribute manager can't record a remote node's attributes to the CIB until +it knows the node is remote. Normally, this is learned when the remote node +starts, because the controller clears the CRM_OP_PROBED attribute and indicates +that it is for a remote node. + +However, if a cluster node is down when a remote node starts, and later comes +up, it learns the remote node's existing attributes as part of the attribute +sync. Previously, this did not include whether each value is for a cluster or +remote node, so the newly joined attribute manager couldn't write out remote +nodes' attributes until it learned that via some other event -- which might not +happen before the node becomes DC, in which case its scheduler will not see any +unfencing-related node attributes and may wrongly schedule unfencing. + +The sync response handling already calls attrd_lookup_or_create_value(), which +checks PCMK__XA_ATTR_IS_REMOTE, so all we need to do is add that to the sync +response. +--- + daemons/attrd/attrd_commands.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c +index 5b32a77..0142383 100644 +--- a/daemons/attrd/attrd_commands.c ++++ b/daemons/attrd/attrd_commands.c +@@ -43,8 +43,9 @@ + * 1 1.1.15 PCMK__ATTRD_CMD_UPDATE_BOTH, + * PCMK__ATTRD_CMD_UPDATE_DELAY + * 2 1.1.17 PCMK__ATTRD_CMD_CLEAR_FAILURE ++ * 3 2.1.1 PCMK__ATTRD_CMD_SYNC_RESPONSE indicates remote nodes + */ +-#define ATTRD_PROTOCOL_VERSION "2" ++#define ATTRD_PROTOCOL_VERSION "3" + + int last_cib_op_done = 0; + GHashTable *attributes = NULL; +@@ -150,6 +151,9 @@ add_attribute_value_xml(xmlNode *parent, attribute_t *a, attribute_value_t *v, + if (v->nodeid > 0) { + crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, v->nodeid); + } ++ if (v->is_remote != 0) { ++ crm_xml_add_int(xml, PCMK__XA_ATTR_IS_REMOTE, 1); ++ } + crm_xml_add(xml, PCMK__XA_ATTR_VALUE, v->current); + crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, a->timeout_ms / 1000); + crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, a->is_private); +-- +1.8.3.1 + diff --git a/SOURCES/043-retry-metadata.patch b/SOURCES/043-retry-metadata.patch new file mode 100644 index 0000000..f66817f --- /dev/null +++ b/SOURCES/043-retry-metadata.patch @@ -0,0 +1,176 @@ +From 5c2d8665773254ff8b9676ac359a1210e34640e3 Mon Sep 17 00:00:00 2001 +From: Oyvind Albrigtsen +Date: Mon, 1 Mar 2021 14:02:52 +0100 +Subject: [PATCH] API: add pcmk__mainloop_timer_get_period() to internal API + +--- + include/crm/common/internal.h | 1 + + lib/common/mainloop.c | 34 +++++++++++++++++++++++++--------- + 2 files changed, 26 insertions(+), 9 deletions(-) + +diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h +index f69abe8..63bfd2c 100644 +--- a/include/crm/common/internal.h ++++ b/include/crm/common/internal.h +@@ -96,6 +96,7 @@ pcmk__open_devnull(int flags) + int pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata, + struct ipc_client_callbacks *callbacks, + mainloop_io_t **source); ++guint pcmk__mainloop_timer_get_period(mainloop_timer_t *timer); + + + /* internal messaging utilities (from messages.c) */ +diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c +index 2f00e31..75f24e2 100644 +--- a/lib/common/mainloop.c ++++ b/lib/common/mainloop.c +@@ -49,6 +49,15 @@ struct trigger_s { + + }; + ++struct mainloop_timer_s { ++ guint id; ++ guint period_ms; ++ bool repeat; ++ char *name; ++ GSourceFunc cb; ++ void *userdata; ++}; ++ + static gboolean + crm_trigger_prepare(GSource * source, gint * timeout) + { +@@ -875,6 +884,22 @@ pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata, + return pcmk_rc_ok; + } + ++/*! ++ * \brief Get period for mainloop timer ++ * ++ * \param[in] timer Timer ++ * ++ * \return Period in ms ++ */ ++guint ++pcmk__mainloop_timer_get_period(mainloop_timer_t *timer) ++{ ++ if (timer) { ++ return timer->period_ms; ++ } ++ return 0; ++} ++ + mainloop_io_t * + mainloop_add_ipc_client(const char *name, int priority, size_t max_size, + void *userdata, struct ipc_client_callbacks *callbacks) +@@ -1252,15 +1277,6 @@ mainloop_child_add(pid_t pid, int timeout, const char *desc, void *privatedata, + mainloop_child_add_with_flags(pid, timeout, desc, privatedata, 0, callback); + } + +-struct mainloop_timer_s { +- guint id; +- guint period_ms; +- bool repeat; +- char *name; +- GSourceFunc cb; +- void *userdata; +-}; +- + static gboolean + mainloop_timer_cb(gpointer user_data) + { +-- +1.8.3.1 + +From 1d33712201e42f0e8ee108999cd4cb8fa0eeca95 Mon Sep 17 00:00:00 2001 +From: Oyvind Albrigtsen +Date: Fri, 19 Feb 2021 12:34:04 +0100 +Subject: [PATCH] Feature: fenced: retry getting metadata until we get it + +--- + daemons/fenced/fenced_commands.c | 35 +++++++++++++++++++++++++++++++++++ + daemons/fenced/pacemaker-fenced.h | 1 + + 2 files changed, 36 insertions(+) + +diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c +index 41901e5..65b41c5 100644 +--- a/daemons/fenced/fenced_commands.c ++++ b/daemons/fenced/fenced_commands.c +@@ -69,6 +69,9 @@ static void stonith_send_reply(xmlNode * reply, int call_options, const char *re + static void search_devices_record_result(struct device_search_s *search, const char *device, + gboolean can_fence); + ++static xmlNode * get_agent_metadata(const char *agent); ++static void read_action_metadata(stonith_device_t *device); ++ + typedef struct async_command_s { + + int id; +@@ -323,6 +326,25 @@ fork_cb(GPid pid, gpointer user_data) + cmd->activating_on = NULL; + } + ++static int ++get_agent_metadata_cb(gpointer data) { ++ stonith_device_t *device = data; ++ ++ device->agent_metadata = get_agent_metadata(device->agent); ++ if (device->agent_metadata) { ++ read_action_metadata(device); ++ stonith__device_parameter_flags(&(device->flags), device->id, ++ device->agent_metadata); ++ return G_SOURCE_REMOVE; ++ } else { ++ guint period_ms = pcmk__mainloop_timer_get_period(device->timer); ++ if (period_ms < 160 * 1000) { ++ mainloop_timer_set_period(device->timer, 2 * period_ms); ++ } ++ return G_SOURCE_CONTINUE; ++ } ++} ++ + static gboolean + stonith_device_execute(stonith_device_t * device) + { +@@ -569,6 +591,11 @@ free_device(gpointer data) + + g_list_free_full(device->targets, free); + ++ if (device->timer) { ++ mainloop_timer_stop(device->timer); ++ mainloop_timer_del(device->timer); ++ } ++ + mainloop_destroy_trigger(device->work); + + free_xml(device->agent_metadata); +@@ -916,6 +943,14 @@ build_device_from_xml(xmlNode * msg) + read_action_metadata(device); + stonith__device_parameter_flags(&(device->flags), device->id, + device->agent_metadata); ++ } else { ++ if (device->timer == NULL) { ++ device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000, ++ TRUE, get_agent_metadata_cb, device); ++ } ++ if (!mainloop_timer_running(device->timer)) { ++ mainloop_timer_start(device->timer); ++ } + } + + value = g_hash_table_lookup(device->params, "nodeid"); +diff --git a/daemons/fenced/pacemaker-fenced.h b/daemons/fenced/pacemaker-fenced.h +index 13cf6dc..e342692 100644 +--- a/daemons/fenced/pacemaker-fenced.h ++++ b/daemons/fenced/pacemaker-fenced.h +@@ -41,6 +41,7 @@ typedef struct stonith_device_s { + GHashTable *params; + GHashTable *aliases; + GList *pending_ops; ++ mainloop_timer_t *timer; + crm_trigger_t *work; + xmlNode *agent_metadata; + +-- +1.8.3.1 + diff --git a/SOURCES/044-sbd.patch b/SOURCES/044-sbd.patch new file mode 100644 index 0000000..f4c7358 --- /dev/null +++ b/SOURCES/044-sbd.patch @@ -0,0 +1,1633 @@ +From 30c04b0f6d717ad27601477eb4b4c47402f46b57 Mon Sep 17 00:00:00 2001 +From: Kazunori INOUE +Date: Fri, 29 Jan 2021 11:28:20 +0900 +Subject: [PATCH] Fix: fencing: remove any devices that are not installed + +--- + daemons/fenced/fenced_commands.c | 2 ++ + daemons/fenced/pacemaker-fenced.c | 37 ++++++++++++++++++++++--------------- + daemons/fenced/pacemaker-fenced.h | 1 + + 3 files changed, 25 insertions(+), 15 deletions(-) + +diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c +index 2729d0d..a4f92cc 100644 +--- a/daemons/fenced/fenced_commands.c ++++ b/daemons/fenced/fenced_commands.c +@@ -1173,6 +1173,8 @@ stonith_device_register(xmlNode * msg, const char **desc, gboolean from_cib) + g_hash_table_size(device_list)); + free_device(device); + device = dup; ++ dup = g_hash_table_lookup(device_list, device->id); ++ dup->dirty = FALSE; + + } else { + stonith_device_t *old = g_hash_table_lookup(device_list, device->id); +diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c +index 5390d66..edfd407 100644 +--- a/daemons/fenced/pacemaker-fenced.c ++++ b/daemons/fenced/pacemaker-fenced.c +@@ -583,11 +583,8 @@ static void cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set) + const char *value = NULL; + const char *rclass = NULL; + pe_node_t *parent = NULL; +- gboolean remove = TRUE; + +- /* If this is a complex resource, check children rather than this resource itself. +- * TODO: Mark each installed device and remove if untouched when this process finishes. +- */ ++ /* If this is a complex resource, check children rather than this resource itself. */ + if(rsc->children) { + GListPtr gIter = NULL; + for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { +@@ -606,10 +603,10 @@ static void cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set) + return; + } + +- /* If this STONITH resource is disabled, just remove it. */ ++ /* If this STONITH resource is disabled, remove it. */ + if (pe__resource_is_disabled(rsc)) { + crm_info("Device %s has been disabled", rsc->id); +- goto update_done; ++ return; + } + + /* Check whether our node is allowed for this resource (and its parent if in a group) */ +@@ -628,7 +625,7 @@ static void cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set) + crm_trace("Available: %s = %d", node->details->uname, node->weight); + } + +- goto update_done; ++ return; + + } else if(node->weight < 0 || (parent && parent->weight < 0)) { + /* Our node (or its group) is disallowed by score, so remove the device */ +@@ -637,7 +634,7 @@ static void cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set) + crm_info("Device %s has been disabled on %s: score=%s", rsc->id, stonith_our_uname, score); + free(score); + +- goto update_done; ++ return; + + } else { + /* Our node is allowed, so update the device information */ +@@ -666,7 +663,6 @@ static void cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set) + crm_trace(" %s=%s", name, value); + } + +- remove = FALSE; + data = create_device_registration_xml(rsc_name(rsc), st_namespace_any, + agent, params, rsc_provides); + stonith_key_value_freeall(params, 1, 1); +@@ -674,12 +670,6 @@ static void cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set) + CRM_ASSERT(rc == pcmk_ok); + free_xml(data); + } +- +-update_done: +- +- if(remove && g_hash_table_lookup(device_list, rsc_name(rsc))) { +- stonith_device_remove(rsc_name(rsc), TRUE); +- } + } + + /*! +@@ -690,6 +680,8 @@ static void + cib_devices_update(void) + { + GListPtr gIter = NULL; ++ GHashTableIter iter; ++ stonith_device_t *device = NULL; + + crm_info("Updating devices to version %s.%s.%s", + crm_element_value(local_cib, XML_ATTR_GENERATION_ADMIN), +@@ -705,9 +697,24 @@ cib_devices_update(void) + cluster_status(fenced_data_set); + pcmk__schedule_actions(fenced_data_set, NULL, NULL); + ++ g_hash_table_iter_init(&iter, device_list); ++ while (g_hash_table_iter_next(&iter, NULL, (void **)&device)) { ++ if (device->cib_registered) { ++ device->dirty = TRUE; ++ } ++ } ++ + for (gIter = fenced_data_set->resources; gIter != NULL; gIter = gIter->next) { + cib_device_update(gIter->data, fenced_data_set); + } ++ ++ g_hash_table_iter_init(&iter, device_list); ++ while (g_hash_table_iter_next(&iter, NULL, (void **)&device)) { ++ if (device->dirty) { ++ g_hash_table_iter_remove(&iter); ++ } ++ } ++ + fenced_data_set->input = NULL; // Wasn't a copy, so don't let API free it + pe_reset_working_set(fenced_data_set); + } +diff --git a/daemons/fenced/pacemaker-fenced.h b/daemons/fenced/pacemaker-fenced.h +index ed2817f..13cf6dc 100644 +--- a/daemons/fenced/pacemaker-fenced.h ++++ b/daemons/fenced/pacemaker-fenced.h +@@ -50,6 +50,7 @@ typedef struct stonith_device_s { + + gboolean cib_registered; + gboolean api_registered; ++ gboolean dirty; + } stonith_device_t; + + /* These values are used to index certain arrays by "phase". Usually an +-- +1.8.3.1 + +From 66a88740105bde4de358f9c1e774ebd5eef3bb68 Mon Sep 17 00:00:00 2001 +From: Kazunori INOUE +Date: Mon, 10 May 2021 12:16:23 +0900 +Subject: [PATCH] Fix: fencing: register/remove the watchdog device + +--- + daemons/fenced/pacemaker-fenced.c | 108 ++++++++++++++++++++++---------------- + 1 file changed, 62 insertions(+), 46 deletions(-) + +diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c +index 2c61423..b05b085 100644 +--- a/daemons/fenced/pacemaker-fenced.c ++++ b/daemons/fenced/pacemaker-fenced.c +@@ -578,6 +578,66 @@ our_node_allowed_for(pe_resource_t *rsc) + return node; + } + ++static void ++watchdog_device_update(xmlNode *cib) ++{ ++ xmlNode *stonith_enabled_xml = NULL; ++ const char *stonith_enabled_s = NULL; ++ long timeout_ms = 0; ++ ++ stonith_enabled_xml = get_xpath_object("//nvpair[@name='stonith-enabled']", ++ cib, LOG_NEVER); ++ if (stonith_enabled_xml) { ++ stonith_enabled_s = crm_element_value(stonith_enabled_xml, XML_NVPAIR_ATTR_VALUE); ++ } ++ ++ if (stonith_enabled_s == NULL || crm_is_true(stonith_enabled_s)) { ++ xmlNode *stonith_watchdog_xml = NULL; ++ const char *value = NULL; ++ ++ stonith_watchdog_xml = get_xpath_object("//nvpair[@name='stonith-watchdog-timeout']", ++ cib, LOG_NEVER); ++ if (stonith_watchdog_xml) { ++ value = crm_element_value(stonith_watchdog_xml, XML_NVPAIR_ATTR_VALUE); ++ } ++ if (value) { ++ timeout_ms = crm_get_msec(value); ++ } ++ ++ if (timeout_ms < 0) { ++ timeout_ms = pcmk__auto_watchdog_timeout(); ++ } ++ } ++ ++ if (timeout_ms != stonith_watchdog_timeout_ms) { ++ crm_notice("New watchdog timeout %lds (was %lds)", timeout_ms/1000, stonith_watchdog_timeout_ms/1000); ++ stonith_watchdog_timeout_ms = timeout_ms; ++ ++ if (stonith_watchdog_timeout_ms > 0) { ++ int rc; ++ xmlNode *xml; ++ stonith_key_value_t *params = NULL; ++ ++ params = stonith_key_value_add(params, PCMK_STONITH_HOST_LIST, ++ stonith_our_uname); ++ ++ xml = create_device_registration_xml("watchdog", st_namespace_internal, ++ STONITH_WATCHDOG_AGENT, params, ++ NULL); ++ stonith_key_value_freeall(params, 1, 1); ++ rc = stonith_device_register(xml, NULL, FALSE); ++ free_xml(xml); ++ if (rc != pcmk_ok) { ++ crm_crit("Cannot register watchdog pseudo fence agent"); ++ crm_exit(CRM_EX_FATAL); ++ } ++ ++ } else { ++ stonith_device_remove("watchdog", FALSE); ++ } ++ } ++} ++ + /*! + * \internal + * \brief If a resource or any of its children are STONITH devices, update their +@@ -1012,7 +1072,6 @@ update_cib_cache_cb(const char *event, xmlNode * msg) + { + int rc = pcmk_ok; + xmlNode *stonith_enabled_xml = NULL; +- xmlNode *stonith_watchdog_xml = NULL; + const char *stonith_enabled_s = NULL; + static gboolean stonith_enabled_saved = TRUE; + +@@ -1076,31 +1135,7 @@ update_cib_cache_cb(const char *event, xmlNode * msg) + stonith_enabled_s = crm_element_value(stonith_enabled_xml, XML_NVPAIR_ATTR_VALUE); + } + +- if (stonith_enabled_s == NULL || crm_is_true(stonith_enabled_s)) { +- long timeout_ms = 0; +- const char *value = NULL; +- +- stonith_watchdog_xml = get_xpath_object("//nvpair[@name='stonith-watchdog-timeout']", +- local_cib, LOG_NEVER); +- if (stonith_watchdog_xml) { +- value = crm_element_value(stonith_watchdog_xml, XML_NVPAIR_ATTR_VALUE); +- } +- +- if(value) { +- timeout_ms = crm_get_msec(value); +- } +- if (timeout_ms < 0) { +- timeout_ms = pcmk__auto_watchdog_timeout(); +- } +- +- if(timeout_ms != stonith_watchdog_timeout_ms) { +- crm_notice("New watchdog timeout %lds (was %lds)", timeout_ms/1000, stonith_watchdog_timeout_ms/1000); +- stonith_watchdog_timeout_ms = timeout_ms; +- } +- +- } else { +- stonith_watchdog_timeout_ms = 0; +- } ++ watchdog_device_update(local_cib); + + if (stonith_enabled_s && crm_is_true(stonith_enabled_s) == FALSE) { + crm_trace("Ignoring cib updates while stonith is disabled"); +@@ -1130,6 +1165,7 @@ init_cib_cache_cb(xmlNode * msg, int call_id, int rc, xmlNode * output, void *us + crm_peer_caches_refresh(local_cib); + + fencing_topology_init(); ++ watchdog_device_update(local_cib); + cib_devices_update(); + } + +@@ -1535,26 +1571,6 @@ main(int argc, char **argv) + init_device_list(); + init_topology_list(); + +- if(stonith_watchdog_timeout_ms > 0) { +- int rc; +- xmlNode *xml; +- stonith_key_value_t *params = NULL; +- +- params = stonith_key_value_add(params, PCMK_STONITH_HOST_LIST, +- stonith_our_uname); +- +- xml = create_device_registration_xml("watchdog", st_namespace_internal, +- STONITH_WATCHDOG_AGENT, params, +- NULL); +- stonith_key_value_freeall(params, 1, 1); +- rc = stonith_device_register(xml, NULL, FALSE); +- free_xml(xml); +- if (rc != pcmk_ok) { +- crm_crit("Cannot register watchdog pseudo fence agent"); +- crm_exit(CRM_EX_FATAL); +- } +- } +- + pcmk__serve_fenced_ipc(&ipcs, &ipc_callbacks); + + /* Create the mainloop and run it... */ +-- +1.8.3.1 + +From b49f49576ef9d801a48ce7a01a78c72e65be7880 Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Fri, 30 Jul 2021 18:07:25 +0200 +Subject: [PATCH 1/3] Fix, Refactor: fenced: add return value to + get_agent_metadata + +Used to distinguish between empty metadata per design, +case of failed getting metadata that might succeed on a +retry and fatal failure. +Fixes as well regression that leads to endless retries getting +metadata for #watchdog - not superserious as it happens with +delays in between but still undesirable. +--- + daemons/fenced/fenced_commands.c | 92 +++++++++++++++++++------------- + 1 file changed, 55 insertions(+), 37 deletions(-) + +diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c +index a778801b1..cd9968f1a 100644 +--- a/daemons/fenced/fenced_commands.c ++++ b/daemons/fenced/fenced_commands.c +@@ -69,7 +69,7 @@ static void stonith_send_reply(xmlNode * reply, int call_options, const char *re + static void search_devices_record_result(struct device_search_s *search, const char *device, + gboolean can_fence); + +-static xmlNode * get_agent_metadata(const char *agent); ++static int get_agent_metadata(const char *agent, xmlNode **metadata); + static void read_action_metadata(stonith_device_t *device); + + typedef struct async_command_s { +@@ -323,19 +323,26 @@ fork_cb(GPid pid, gpointer user_data) + static int + get_agent_metadata_cb(gpointer data) { + stonith_device_t *device = data; ++ guint period_ms; + +- device->agent_metadata = get_agent_metadata(device->agent); +- if (device->agent_metadata) { +- read_action_metadata(device); +- stonith__device_parameter_flags(&(device->flags), device->id, ++ switch (get_agent_metadata(device->agent, &device->agent_metadata)) { ++ case pcmk_rc_ok: ++ if (device->agent_metadata) { ++ read_action_metadata(device); ++ stonith__device_parameter_flags(&(device->flags), device->id, + device->agent_metadata); +- return G_SOURCE_REMOVE; +- } else { +- guint period_ms = pcmk__mainloop_timer_get_period(device->timer); +- if (period_ms < 160 * 1000) { +- mainloop_timer_set_period(device->timer, 2 * period_ms); +- } +- return G_SOURCE_CONTINUE; ++ } ++ return G_SOURCE_REMOVE; ++ ++ case EAGAIN: ++ period_ms = pcmk__mainloop_timer_get_period(device->timer); ++ if (period_ms < 160 * 1000) { ++ mainloop_timer_set_period(device->timer, 2 * period_ms); ++ } ++ return G_SOURCE_CONTINUE; ++ ++ default: ++ return G_SOURCE_REMOVE; + } + } + +@@ -700,38 +707,41 @@ init_metadata_cache(void) { + } + } + +-static xmlNode * +-get_agent_metadata(const char *agent) ++int ++get_agent_metadata(const char *agent, xmlNode ** metadata) + { +- xmlNode *xml = NULL; + char *buffer = NULL; + ++ if (metadata == NULL) { ++ return EINVAL; ++ } ++ *metadata = NULL; ++ if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT, pcmk__str_none)) { ++ return pcmk_rc_ok; ++ } + init_metadata_cache(); + buffer = g_hash_table_lookup(metadata_cache, agent); +- if(pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT, pcmk__str_casei)) { +- return NULL; +- +- } else if(buffer == NULL) { ++ if (buffer == NULL) { + stonith_t *st = stonith_api_new(); + int rc; + + if (st == NULL) { + crm_warn("Could not get agent meta-data: " + "API memory allocation failed"); +- return NULL; ++ return EAGAIN; + } +- rc = st->cmds->metadata(st, st_opt_sync_call, agent, NULL, &buffer, 10); ++ rc = st->cmds->metadata(st, st_opt_sync_call, agent, ++ NULL, &buffer, 10); + stonith_api_delete(st); + if (rc || !buffer) { + crm_err("Could not retrieve metadata for fencing agent %s", agent); +- return NULL; ++ return EAGAIN; + } + g_hash_table_replace(metadata_cache, strdup(agent), buffer); + } + +- xml = string2xml(buffer); +- +- return xml; ++ *metadata = string2xml(buffer); ++ return pcmk_rc_ok; + } + + static gboolean +@@ -962,19 +972,27 @@ build_device_from_xml(xmlNode * msg) + value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP); + device->aliases = build_port_aliases(value, &(device->targets)); + +- device->agent_metadata = get_agent_metadata(device->agent); +- if (device->agent_metadata) { +- read_action_metadata(device); +- stonith__device_parameter_flags(&(device->flags), device->id, +- device->agent_metadata); +- } else { +- if (device->timer == NULL) { +- device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000, ++ switch (get_agent_metadata(device->agent, &device->agent_metadata)) { ++ case pcmk_rc_ok: ++ if (device->agent_metadata) { ++ read_action_metadata(device); ++ stonith__device_parameter_flags(&(device->flags), device->id, ++ device->agent_metadata); ++ } ++ break; ++ ++ case EAGAIN: ++ if (device->timer == NULL) { ++ device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000, + TRUE, get_agent_metadata_cb, device); +- } +- if (!mainloop_timer_running(device->timer)) { +- mainloop_timer_start(device->timer); +- } ++ } ++ if (!mainloop_timer_running(device->timer)) { ++ mainloop_timer_start(device->timer); ++ } ++ break; ++ ++ default: ++ break; + } + + value = g_hash_table_lookup(device->params, "nodeid"); +-- +2.27.0 + + +From 5dd1e4459335764e0adf5fa78d81c875ae2332e9 Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Fri, 30 Jul 2021 18:15:10 +0200 +Subject: [PATCH 2/3] feature: watchdog-fencing: allow restriction to certain + nodes + +Bump CRM_FEATURE_SET to 3.11.0 to encourage cluster being +fully upgraded to a version that supports the feature +before explicitly adding a watchdog-fence-device. +--- + configure.ac | 1 + + daemons/controld/controld_control.c | 2 +- + daemons/controld/controld_fencing.c | 14 ++ + daemons/controld/controld_fencing.h | 1 + + daemons/fenced/Makefile.am | 2 +- + daemons/fenced/fence_watchdog.in | 283 ++++++++++++++++++++++++++++ + daemons/fenced/fenced_commands.c | 141 +++++++++++--- + daemons/fenced/fenced_remote.c | 71 ++++--- + daemons/fenced/pacemaker-fenced.c | 131 +++++++++---- + daemons/fenced/pacemaker-fenced.h | 5 +- + include/crm/crm.h | 2 +- + include/crm/fencing/internal.h | 8 +- + lib/fencing/st_client.c | 61 ++++++ + lib/lrmd/lrmd_client.c | 6 +- + rpm/pacemaker.spec.in | 3 + + 16 files changed, 635 insertions(+), 97 deletions(-) + create mode 100755 daemons/fenced/fence_watchdog.in + +diff --git a/configure.ac b/configure.ac +index 436100c81..013562e46 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1950,6 +1950,7 @@ + AC_CONFIG_FILES([cts/fence_dummy], [chmod +x cts/fence_dummy]) + AC_CONFIG_FILES([cts/pacemaker-cts-dummyd], [chmod +x cts/pacemaker-cts-dummyd]) + AC_CONFIG_FILES([daemons/fenced/fence_legacy], [chmod +x daemons/fenced/fence_legacy]) ++AC_CONFIG_FILES([daemons/fenced/fence_watchdog], [chmod +x daemons/fenced/fence_watchdog]) + AC_CONFIG_FILES([doc/abi-check], [chmod +x doc/abi-check]) + AC_CONFIG_FILES([extra/resources/ClusterMon], [chmod +x extra/resources/ClusterMon]) + AC_CONFIG_FILES([extra/resources/HealthSMART], [chmod +x extra/resources/HealthSMART]) +diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c +index 45a70bb92..b5da6a46c 100644 +--- a/daemons/controld/controld_control.c ++++ b/daemons/controld/controld_control.c +@@ -615,7 +615,7 @@ static pcmk__cluster_option_t crmd_opts[] = { + }, + { + "stonith-watchdog-timeout", NULL, "time", NULL, +- "0", pcmk__valid_sbd_timeout, ++ "0", controld_verify_stonith_watchdog_timeout, + "How long to wait before we can assume nodes are safely down " + "when watchdog-based self-fencing via SBD is in use", + "If nonzero, along with `have-watchdog=true` automatically set by the " +diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c +index 0fba6613b..6c2a6c550 100644 +--- a/daemons/controld/controld_fencing.c ++++ b/daemons/controld/controld_fencing.c +@@ -11,6 +11,7 @@ + #include + #include + #include ++#include + #include + + #include +@@ -886,6 +887,19 @@ te_fence_node(crm_graph_t *graph, crm_action_t *action) + return TRUE; + } + ++bool ++controld_verify_stonith_watchdog_timeout(const char *value) ++{ ++ gboolean rv = TRUE; ++ ++ if (stonith_api && (stonith_api->state != stonith_disconnected) && ++ stonith__watchdog_fencing_enabled_for_node_api(stonith_api, ++ fsa_our_uname)) { ++ rv = pcmk__valid_sbd_timeout(value); ++ } ++ return rv; ++} ++ + /* end stonith API client functions */ + + +diff --git a/daemons/controld/controld_fencing.h b/daemons/controld/controld_fencing.h +index d0ecc8234..ef68a0c83 100644 +--- a/daemons/controld/controld_fencing.h ++++ b/daemons/controld/controld_fencing.h +@@ -24,6 +24,7 @@ void update_stonith_max_attempts(const char* value); + void controld_trigger_fencer_connect(void); + void controld_disconnect_fencer(bool destroy); + gboolean te_fence_node(crm_graph_t *graph, crm_action_t *action); ++bool controld_verify_stonith_watchdog_timeout(const char *value); + + // stonith cleanup list + void add_stonith_cleanup(const char *target); +diff --git a/daemons/fenced/Makefile.am b/daemons/fenced/Makefile.am +index 43413e11d..2923d7c9b 100644 +--- a/daemons/fenced/Makefile.am ++++ b/daemons/fenced/Makefile.am +@@ -15,7 +15,7 @@ halibdir = $(CRM_DAEMON_DIR) + + halib_PROGRAMS = pacemaker-fenced cts-fence-helper + +-sbin_SCRIPTS = fence_legacy ++sbin_SCRIPTS = fence_legacy fence_watchdog + + noinst_HEADERS = pacemaker-fenced.h + +diff --git a/daemons/fenced/fence_watchdog.in b/daemons/fenced/fence_watchdog.in +new file mode 100755 +index 000000000..c83304f1d +--- /dev/null ++++ b/daemons/fenced/fence_watchdog.in +@@ -0,0 +1,284 @@ ++#!@PYTHON@ ++"""Dummy watchdog fence agent for providing meta-data for the pacemaker internal agent ++""" ++ ++__copyright__ = "Copyright 2012-2021 the Pacemaker project contributors" ++__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" ++ ++import io ++import os ++import re ++import sys ++import atexit ++import getopt ++ ++AGENT_VERSION = "1.0.0" ++SHORT_DESC = "Dummy watchdog fence agent" ++LONG_DESC = """fence_watchdog just provides ++meta-data - actual fencing is done by the pacemaker internal watchdog agent.""" ++ ++ALL_OPT = { ++ "version" : { ++ "getopt" : "V", ++ "longopt" : "version", ++ "help" : "-V, --version Display version information and exit", ++ "required" : "0", ++ "shortdesc" : "Display version information and exit", ++ "order" : 53 ++ }, ++ "help" : { ++ "getopt" : "h", ++ "longopt" : "help", ++ "help" : "-h, --help Display this help and exit", ++ "required" : "0", ++ "shortdesc" : "Display help and exit", ++ "order" : 54 ++ }, ++ "action" : { ++ "getopt" : "o:", ++ "longopt" : "action", ++ "help" : "-o, --action=[action] Action: metadata", ++ "required" : "1", ++ "shortdesc" : "Fencing Action", ++ "default" : "metadata", ++ "order" : 1 ++ }, ++ "nodename" : { ++ "getopt" : "N:", ++ "longopt" : "nodename", ++ "help" : "-N, --nodename Node name of fence victim (ignored)", ++ "required" : "0", ++ "shortdesc" : "Ignored", ++ "order" : 2 ++ }, ++ "plug" : { ++ "getopt" : "n:", ++ "longopt" : "plug", ++ "help" : "-n, --plug=[id] Physical plug number on device (ignored)", ++ "required" : "1", ++ "shortdesc" : "Ignored", ++ "order" : 4 ++ } ++} ++ ++ ++def agent(): ++ """ Return name this file was run as. """ ++ ++ return os.path.basename(sys.argv[0]) ++ ++ ++def fail_usage(message): ++ """ Print a usage message and exit. """ ++ ++ sys.exit("%s\nPlease use '-h' for usage" % message) ++ ++ ++def show_docs(options): ++ """ Handle informational options (display info and exit). """ ++ ++ device_opt = options["device_opt"] ++ ++ if "-h" in options: ++ usage(device_opt) ++ sys.exit(0) ++ ++ if "-o" in options and options["-o"].lower() == "metadata": ++ metadata(device_opt, options) ++ sys.exit(0) ++ ++ if "-V" in options: ++ print(AGENT_VERSION) ++ sys.exit(0) ++ ++ ++def sorted_options(avail_opt): ++ """ Return a list of all options, in their internally specified order. """ ++ ++ sorted_list = [(key, ALL_OPT[key]) for key in avail_opt] ++ sorted_list.sort(key=lambda x: x[1]["order"]) ++ return sorted_list ++ ++ ++def usage(avail_opt): ++ """ Print a usage message. """ ++ print(LONG_DESC) ++ print() ++ print("Usage:") ++ print("\t" + agent() + " [options]") ++ print("Options:") ++ ++ for dummy, value in sorted_options(avail_opt): ++ if len(value["help"]) != 0: ++ print(" " + value["help"]) ++ ++ ++def metadata(avail_opt, options): ++ """ Print agent metadata. """ ++ ++ print(""" ++ ++%s ++""" % (agent(), SHORT_DESC, LONG_DESC)) ++ ++ for option, dummy in sorted_options(avail_opt): ++ if "shortdesc" in ALL_OPT[option]: ++ print(' ') ++ ++ default = "" ++ default_name_arg = "-" + ALL_OPT[option]["getopt"][:-1] ++ default_name_no_arg = "-" + ALL_OPT[option]["getopt"] ++ ++ if "default" in ALL_OPT[option]: ++ default = 'default="%s"' % str(ALL_OPT[option]["default"]) ++ elif default_name_arg in options: ++ if options[default_name_arg]: ++ try: ++ default = 'default="%s"' % options[default_name_arg] ++ except TypeError: ++ ## @todo/@note: Currently there is no clean way how to handle lists ++ ## we can create a string from it but we can't set it on command line ++ default = 'default="%s"' % str(options[default_name_arg]) ++ elif default_name_no_arg in options: ++ default = 'default="true"' ++ ++ mixed = ALL_OPT[option]["help"] ++ ## split it between option and help text ++ res = re.compile(r"^(.*--\S+)\s+", re.IGNORECASE | re.S).search(mixed) ++ if None != res: ++ mixed = res.group(1) ++ mixed = mixed.replace("<", "<").replace(">", ">") ++ print(' ') ++ ++ if ALL_OPT[option]["getopt"].count(":") > 0: ++ print(' ') ++ else: ++ print(' ') ++ ++ print(' ' + ALL_OPT[option]["shortdesc"] + '') ++ print(' ') ++ ++ print(' \n ') ++ print(' ') ++ print(' ') ++ print(' ') ++ print(' ') ++ print(' ') ++ print(' ') ++ print(' ') ++ print('') ++ ++ ++def option_longopt(option): ++ """ Return the getopt-compatible long-option name of the given option. """ ++ ++ if ALL_OPT[option]["getopt"].endswith(":"): ++ return ALL_OPT[option]["longopt"] + "=" ++ else: ++ return ALL_OPT[option]["longopt"] ++ ++ ++def opts_from_command_line(argv, avail_opt): ++ """ Read options from command-line arguments. """ ++ ++ # Prepare list of options for getopt ++ getopt_string = "" ++ longopt_list = [] ++ for k in avail_opt: ++ if k in ALL_OPT: ++ getopt_string += ALL_OPT[k]["getopt"] ++ else: ++ fail_usage("Parse error: unknown option '" + k + "'") ++ ++ if k in ALL_OPT and "longopt" in ALL_OPT[k]: ++ longopt_list.append(option_longopt(k)) ++ ++ try: ++ opt, dummy = getopt.gnu_getopt(argv, getopt_string, longopt_list) ++ except getopt.GetoptError as error: ++ fail_usage("Parse error: " + error.msg) ++ ++ # Transform longopt to short one which are used in fencing agents ++ old_opt = opt ++ opt = {} ++ for old_option in dict(old_opt).keys(): ++ if old_option.startswith("--"): ++ for option in ALL_OPT.keys(): ++ if "longopt" in ALL_OPT[option] and "--" + ALL_OPT[option]["longopt"] == old_option: ++ opt["-" + ALL_OPT[option]["getopt"].rstrip(":")] = dict(old_opt)[old_option] ++ else: ++ opt[old_option] = dict(old_opt)[old_option] ++ ++ return opt ++ ++ ++def opts_from_stdin(avail_opt): ++ """ Read options from standard input. """ ++ ++ opt = {} ++ name = "" ++ for line in sys.stdin.readlines(): ++ line = line.strip() ++ if line.startswith("#") or (len(line) == 0): ++ continue ++ ++ (name, value) = (line + "=").split("=", 1) ++ value = value[:-1] ++ ++ if name not in avail_opt: ++ print("Parse error: Ignoring unknown option '%s'" % line, ++ file=sys.stderr) ++ continue ++ ++ if ALL_OPT[name]["getopt"].endswith(":"): ++ opt["-"+ALL_OPT[name]["getopt"].rstrip(":")] = value ++ elif value.lower() in ["1", "yes", "on", "true"]: ++ opt["-"+ALL_OPT[name]["getopt"]] = "1" ++ ++ return opt ++ ++ ++def process_input(avail_opt): ++ """ Set standard environment variables, and parse all options. """ ++ ++ # Set standard environment ++ os.putenv("LANG", "C") ++ os.putenv("LC_ALL", "C") ++ ++ # Read options from command line or standard input ++ if len(sys.argv) > 1: ++ return opts_from_command_line(sys.argv[1:], avail_opt) ++ else: ++ return opts_from_stdin(avail_opt) ++ ++ ++def atexit_handler(): ++ """ Close stdout on exit. """ ++ ++ try: ++ sys.stdout.close() ++ os.close(1) ++ except IOError: ++ sys.exit("%s failed to close standard output" % agent()) ++ ++ ++def main(): ++ """ Make it so! """ ++ ++ device_opt = ALL_OPT.keys() ++ ++ ## Defaults for fence agent ++ atexit.register(atexit_handler) ++ options = process_input(device_opt) ++ options["device_opt"] = device_opt ++ show_docs(options) ++ ++ print("Watchdog fencing may be initiated only by the cluster, not this agent.", ++ file=sys.stderr) ++ ++ sys.exit(1) ++ ++ ++if __name__ == "__main__": ++ main() +diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c +index cd9968f1a..9470ea2c1 100644 +--- a/daemons/fenced/fenced_commands.c ++++ b/daemons/fenced/fenced_commands.c +@@ -397,15 +397,13 @@ stonith_device_execute(stonith_device_t * device) + return TRUE; + } + +- if(pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT, pcmk__str_casei)) { +- if(pcmk__str_eq(cmd->action, "reboot", pcmk__str_casei)) { +- pcmk__panic(__func__); +- goto done; +- +- } else if(pcmk__str_eq(cmd->action, "off", pcmk__str_casei)) { +- pcmk__panic(__func__); +- goto done; +- ++ if (pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT, ++ STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) { ++ if (pcmk__str_any_of(cmd->action, "reboot", "off", NULL)) { ++ if (node_does_watchdog_fencing(stonith_our_uname)) { ++ pcmk__panic(__func__); ++ goto done; ++ } + } else { + crm_info("Faking success for %s watchdog operation", cmd->action); + cmd->done_cb(0, 0, NULL, cmd); +@@ -716,7 +714,7 @@ get_agent_metadata(const char *agent, xmlNode ** metadata) + return EINVAL; + } + *metadata = NULL; +- if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT, pcmk__str_none)) { ++ if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT_INTERNAL, pcmk__str_none)) { + return pcmk_rc_ok; + } + init_metadata_cache(); +@@ -1050,24 +1048,6 @@ schedule_internal_command(const char *origin, + schedule_stonith_command(cmd, device); + } + +-gboolean +-string_in_list(GListPtr list, const char *item) +-{ +- int lpc = 0; +- int max = g_list_length(list); +- +- for (lpc = 0; lpc < max; lpc++) { +- const char *value = g_list_nth_data(list, lpc); +- +- if (pcmk__str_eq(item, value, pcmk__str_casei)) { +- return TRUE; +- } else { +- crm_trace("%d: '%s' != '%s'", lpc, item, value); +- } +- } +- return FALSE; +-} +- + static void + status_search_cb(GPid pid, int rc, const char *output, gpointer user_data) + { +@@ -1144,7 +1124,7 @@ dynamic_list_search_cb(GPid pid, int rc, const char *output, gpointer user_data) + if (!alias) { + alias = search->host; + } +- if (string_in_list(dev->targets, alias)) { ++ if (pcmk__strcase_in_list(dev->targets, alias)) { + can_fence = TRUE; + } + } +@@ -1215,8 +1195,61 @@ stonith_device_register(xmlNode * msg, const char **desc, gboolean from_cib) + stonith_device_t *dup = NULL; + stonith_device_t *device = build_device_from_xml(msg); ++ int rv = pcmk_ok; + + CRM_CHECK(device != NULL, return -ENOMEM); + ++ /* do we have a watchdog-device? */ ++ if (pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none) || ++ pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT, ++ STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) do { ++ if (stonith_watchdog_timeout_ms <= 0) { ++ crm_err("Ignoring watchdog fence device without " ++ "stonith-watchdog-timeout set."); ++ rv = -ENODEV; ++ /* fall through to cleanup & return */ ++ } else if (!pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT, ++ STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) { ++ crm_err("Ignoring watchdog fence device with unknown " ++ "agent '%s' unequal '" STONITH_WATCHDOG_AGENT "'.", ++ device->agent?device->agent:""); ++ rv = -ENODEV; ++ /* fall through to cleanup & return */ ++ } else if (!pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, ++ pcmk__str_none)) { ++ crm_err("Ignoring watchdog fence device " ++ "named %s !='"STONITH_WATCHDOG_ID"'.", ++ device->id?device->id:""); ++ rv = -ENODEV; ++ /* fall through to cleanup & return */ ++ } else { ++ if (pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT, ++ pcmk__str_none)) { ++ /* this either has an empty list or the targets ++ configured for watchdog-fencing ++ */ ++ g_list_free_full(stonith_watchdog_targets, free); ++ stonith_watchdog_targets = device->targets; ++ device->targets = NULL; ++ } ++ if (node_does_watchdog_fencing(stonith_our_uname)) { ++ g_list_free_full(device->targets, free); ++ device->targets = stonith__parse_targets(stonith_our_uname); ++ g_hash_table_replace(device->params, ++ strdup(PCMK_STONITH_HOST_LIST), ++ strdup(stonith_our_uname)); ++ /* proceed as with any other stonith-device */ ++ break; ++ } ++ ++ crm_debug("Skip registration of watchdog fence device on node not in host-list."); ++ /* cleanup and fall through to more cleanup and return */ ++ device->targets = NULL; ++ stonith_device_remove(device->id, from_cib); ++ } ++ free_device(device); ++ return rv; ++ } while (0); ++ + dup = device_has_duplicate(device); + if (dup) { + crm_debug("Device '%s' already existed in device list (%d active devices)", device->id, +@@ -1598,6 +1631,39 @@ stonith_level_remove(xmlNode *msg, char **desc) + * (CIB registration is not sufficient), because monitor should not be + * possible unless the device is "started" (API registered). + */ ++ ++static char * ++list_to_string(GList *list, const char *delim, gboolean terminate_with_delim) ++{ ++ int max = g_list_length(list); ++ size_t delim_len = delim?strlen(delim):0; ++ size_t alloc_size = 1 + (max?((max-1+(terminate_with_delim?1:0))*delim_len):0); ++ char *rv; ++ GList *gIter; ++ ++ for (gIter = list; gIter != NULL; gIter = gIter->next) { ++ const char *value = (const char *) gIter->data; ++ ++ alloc_size += strlen(value); ++ } ++ rv = calloc(alloc_size, sizeof(char)); ++ if (rv) { ++ char *pos = rv; ++ const char *lead_delim = ""; ++ ++ for (gIter = list; gIter != NULL; gIter = gIter->next) { ++ const char *value = (const char *) gIter->data; ++ ++ pos = &pos[sprintf(pos, "%s%s", lead_delim, value)]; ++ lead_delim = delim; ++ } ++ if (max && terminate_with_delim) { ++ sprintf(pos, "%s", delim); ++ } ++ } ++ return rv; ++} ++ + static int + stonith_device_action(xmlNode * msg, char **output) + { +@@ -1615,6 +1681,19 @@ stonith_device_action(xmlNode * msg, char **output) + return -EPROTO; + } + ++ if (pcmk__str_eq(id, STONITH_WATCHDOG_ID, pcmk__str_none)) { ++ if (stonith_watchdog_timeout_ms <= 0) { ++ return -ENODEV; ++ } else { ++ if (pcmk__str_eq(action, "list", pcmk__str_casei)) { ++ *output = list_to_string(stonith_watchdog_targets, "\n", TRUE); ++ return pcmk_ok; ++ } else if (pcmk__str_eq(action, "monitor", pcmk__str_casei)) { ++ return pcmk_ok; ++ } ++ } ++ } ++ + device = g_hash_table_lookup(device_list, id); + if ((device == NULL) + || (!device->api_registered && !strcmp(action, "monitor"))) { +@@ -1742,7 +1821,7 @@ can_fence_host_with_device(stonith_device_t * dev, struct device_search_s *searc + * Only use if all hosts on which the device can be active can always fence all listed hosts + */ + +- if (string_in_list(dev->targets, host)) { ++ if (pcmk__strcase_in_list(dev->targets, host)) { + can = TRUE; + } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP) + && g_hash_table_lookup(dev->aliases, host)) { +@@ -1763,7 +1842,7 @@ can_fence_host_with_device(stonith_device_t * dev, struct device_search_s *searc + return; + } + +- if (string_in_list(dev->targets, alias)) { ++ if (pcmk__strcase_in_list(dev->targets, alias)) { + can = TRUE; + } + +diff --git a/daemons/fenced/fenced_remote.c b/daemons/fenced/fenced_remote.c +index cf91acaed..224f2baba 100644 +--- a/daemons/fenced/fenced_remote.c ++++ b/daemons/fenced/fenced_remote.c +@@ -1522,6 +1522,25 @@ advance_topology_device_in_level(remote_fencing_op_t *op, const char *device, + } + } + ++static gboolean ++check_watchdog_fencing_and_wait(remote_fencing_op_t * op) ++{ ++ if (node_does_watchdog_fencing(op->target)) { ++ ++ crm_notice("Waiting %lds for %s to self-fence (%s) for " ++ "client %s " CRM_XS " id=%.8s", ++ (stonith_watchdog_timeout_ms / 1000), ++ op->target, op->action, op->client_name, op->id); ++ op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, ++ remote_op_watchdog_done, op); ++ return TRUE; ++ } else { ++ crm_debug("Skipping fallback to watchdog-fencing as %s is " ++ "not in host-list", op->target); ++ } ++ return FALSE; ++} ++ + void + call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer, int rc) + { +@@ -1592,26 +1611,33 @@ call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer, int rc) + g_source_remove(op->op_timer_one); + } + +- if(stonith_watchdog_timeout_ms > 0 && device && pcmk__str_eq(device, "watchdog", pcmk__str_casei)) { +- crm_notice("Waiting %lds for %s to self-fence (%s) for client %s.%.8s", +- stonith_watchdog_timeout_ms/1000, op->target, op->action, +- op->client_name, op->id); +- op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, remote_op_watchdog_done, op); +- +- /* TODO check devices to verify watchdog will be in use */ +- } else if(stonith_watchdog_timeout_ms > 0 +- && pcmk__str_eq(peer->host, op->target, pcmk__str_casei) +- && !pcmk__str_eq(op->action, "on", pcmk__str_casei)) { +- crm_notice("Waiting %lds for %s to self-fence (%s) for client %s.%.8s", +- stonith_watchdog_timeout_ms/1000, op->target, op->action, +- op->client_name, op->id); +- op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, remote_op_watchdog_done, op); +- +- } else { ++ if (!(stonith_watchdog_timeout_ms > 0 && ( ++ (pcmk__str_eq(device, STONITH_WATCHDOG_ID, ++ pcmk__str_none)) || ++ (pcmk__str_eq(peer->host, op->target, pcmk__str_casei) ++ && !pcmk__str_eq(op->action, "on", pcmk__str_casei))) && ++ check_watchdog_fencing_and_wait(op))) { ++ ++ /* Some thoughts about self-fencing cases reaching this point: ++ - Actually check in check_watchdog_fencing_and_wait ++ shouldn't fail if STONITH_WATCHDOG_ID is ++ chosen as fencing-device and it being present implies ++ watchdog-fencing is enabled anyway ++ - If watchdog-fencing is disabled either in general or for ++ a specific target - detected in check_watchdog_fencing_and_wait - ++ for some other kind of self-fencing we can't expect ++ a success answer but timeout is fine if the node doesn't ++ come back in between ++ - Delicate might be the case where we have watchdog-fencing ++ enabled for a node but the watchdog-fencing-device isn't ++ explicitly chosen for suicide. Local pe-execution in sbd ++ may detect the node as unclean and lead to timely suicide. ++ Otherwise the selection of stonith-watchdog-timeout at ++ least is questionable. ++ */ + op->op_timer_one = g_timeout_add((1000 * timeout_one), remote_op_timeout_one, op); + } + +- + send_cluster_message(crm_get_peer(0, peer->host), crm_msg_stonith_ng, remote_op, FALSE); + peer->tried = TRUE; + free_xml(remote_op); +@@ -1645,13 +1671,11 @@ call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer, int rc) + * but we have all the expected replies, then no devices + * are available to execute the fencing operation. */ + +- if(stonith_watchdog_timeout_ms && pcmk__str_eq(device, "watchdog", pcmk__str_null_matches | pcmk__str_casei)) { +- crm_notice("Waiting %lds for %s to self-fence (%s) for client %s.%.8s", +- stonith_watchdog_timeout_ms/1000, op->target, +- op->action, op->client_name, op->id); +- +- op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, remote_op_watchdog_done, op); +- return; ++ if(stonith_watchdog_timeout_ms > 0 && pcmk__str_eq(device, ++ STONITH_WATCHDOG_ID, pcmk__str_null_matches)) { ++ if (check_watchdog_fencing_and_wait(op)) { ++ return; ++ } + } + + if (op->state == st_query) { +diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c +index 39738d8be..7f8b427d9 100644 +--- a/daemons/fenced/pacemaker-fenced.c ++++ b/daemons/fenced/pacemaker-fenced.c +@@ -42,7 +42,8 @@ + + char *stonith_our_uname = NULL; + char *stonith_our_uuid = NULL; + long stonith_watchdog_timeout_ms = 0; ++GList *stonith_watchdog_targets = NULL; + + static GMainLoop *mainloop = NULL; + +@@ -578,7 +579,44 @@ our_node_allowed_for(pe_resource_t *rsc) + } + + static void +-watchdog_device_update(xmlNode *cib) ++watchdog_device_update(void) ++{ ++ if (stonith_watchdog_timeout_ms > 0) { ++ if (!g_hash_table_lookup(device_list, STONITH_WATCHDOG_ID) && ++ !stonith_watchdog_targets) { ++ /* getting here watchdog-fencing enabled, no device there yet ++ and reason isn't stonith_watchdog_targets preventing that ++ */ ++ int rc; ++ xmlNode *xml; ++ ++ xml = create_device_registration_xml( ++ STONITH_WATCHDOG_ID, ++ st_namespace_internal, ++ STONITH_WATCHDOG_AGENT, ++ NULL, /* stonith_device_register will add our ++ own name as PCMK_STONITH_HOST_LIST param ++ so we can skip that here ++ */ ++ NULL); ++ rc = stonith_device_register(xml, NULL, TRUE); ++ free_xml(xml); ++ if (rc != pcmk_ok) { ++ crm_crit("Cannot register watchdog pseudo fence agent"); ++ crm_exit(CRM_EX_FATAL); ++ } ++ } ++ ++ } else { ++ /* be silent if no device - todo parameter to stonith_device_remove */ ++ if (g_hash_table_lookup(device_list, STONITH_WATCHDOG_ID)) { ++ stonith_device_remove(STONITH_WATCHDOG_ID, TRUE); ++ } ++ } ++} ++ ++static void ++update_stonith_watchdog_timeout_ms(xmlNode *cib) + { + xmlNode *stonith_enabled_xml = NULL; + const char *stonith_enabled_s = NULL; +@@ -608,33 +646,7 @@ watchdog_device_update(xmlNode *cib) + } + } + +- if (timeout_ms != stonith_watchdog_timeout_ms) { +- crm_notice("New watchdog timeout %lds (was %lds)", timeout_ms/1000, stonith_watchdog_timeout_ms/1000); +- stonith_watchdog_timeout_ms = timeout_ms; +- +- if (stonith_watchdog_timeout_ms > 0) { +- int rc; +- xmlNode *xml; +- stonith_key_value_t *params = NULL; +- +- params = stonith_key_value_add(params, PCMK_STONITH_HOST_LIST, +- stonith_our_uname); +- +- xml = create_device_registration_xml("watchdog", st_namespace_internal, +- STONITH_WATCHDOG_AGENT, params, +- NULL); +- stonith_key_value_freeall(params, 1, 1); +- rc = stonith_device_register(xml, NULL, FALSE); +- free_xml(xml); +- if (rc != pcmk_ok) { +- crm_crit("Cannot register watchdog pseudo fence agent"); +- crm_exit(CRM_EX_FATAL); +- } +- +- } else { +- stonith_device_remove("watchdog", FALSE); +- } +- } ++ stonith_watchdog_timeout_ms = timeout_ms; + } + + /*! +@@ -677,6 +689,16 @@ static void cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set) + return; + } + ++ /* if watchdog-fencing is disabled handle any watchdog-fence ++ resource as if it was disabled ++ */ ++ if ((stonith_watchdog_timeout_ms <= 0) && ++ pcmk__str_eq(rsc->id, STONITH_WATCHDOG_ID, pcmk__str_none)) { ++ crm_info("Watchdog-fencing disabled thus handling " ++ "device %s as disabled", rsc->id); ++ return; ++ } ++ + /* Check whether our node is allowed for this resource (and its parent if in a group) */ + node = our_node_allowed_for(rsc); + if (rsc->parent && (rsc->parent->variant == pe_group)) { +@@ -772,6 +794,12 @@ cib_devices_update(void) + } + } + ++ /* have list repopulated if cib has a watchdog-fencing-resource ++ TODO: keep a cached list for queries happening while we are refreshing ++ */ ++ g_list_free_full(stonith_watchdog_targets, free); ++ stonith_watchdog_targets = NULL; ++ + for (gIter = fenced_data_set->resources; gIter != NULL; gIter = gIter->next) { + cib_device_update(gIter->data, fenced_data_set); + } +@@ -825,6 +853,8 @@ update_cib_stonith_devices_v2(const char *event, xmlNode * msg) + if (search != NULL) { + *search = 0; + stonith_device_remove(rsc_id, TRUE); ++ /* watchdog_device_update called afterwards ++ to fall back to implicit definition if needed */ + } else { + crm_warn("Ignoring malformed CIB update (resource deletion)"); + } +@@ -968,6 +998,24 @@ node_has_attr(const char *node, const char *name, const char *value) + return (match != NULL); + } + ++/*! ++ * \internal ++ * \brief Check whether a node does watchdog-fencing ++ * ++ * \param[in] node Name of node to check ++ * ++ * \return TRUE if node found in stonith_watchdog_targets ++ * or stonith_watchdog_targets is empty indicating ++ * all nodes are doing watchdog-fencing ++ */ ++gboolean ++node_does_watchdog_fencing(const char *node) ++{ ++ return ((stonith_watchdog_targets == NULL) || ++ pcmk__strcase_in_list(stonith_watchdog_targets, node)); ++} ++ ++ + static void + update_fencing_topology(const char *event, xmlNode * msg) + { +@@ -1073,6 +1121,8 @@ update_cib_cache_cb(const char *event, xmlNode * msg) + xmlNode *stonith_enabled_xml = NULL; + const char *stonith_enabled_s = NULL; + static gboolean stonith_enabled_saved = TRUE; ++ long timeout_ms_saved = stonith_watchdog_timeout_ms; ++ gboolean need_full_refresh = FALSE; + + if(!have_cib_devices) { + crm_trace("Skipping updates until we get a full dump"); +@@ -1127,6 +1177,7 @@ update_cib_cache_cb(const char *event, xmlNode * msg) + } + + crm_peer_caches_refresh(local_cib); ++ update_stonith_watchdog_timeout_ms(local_cib); + + stonith_enabled_xml = get_xpath_object("//nvpair[@name='stonith-enabled']", + local_cib, LOG_NEVER); +@@ -1134,22 +1185,29 @@ update_cib_cache_cb(const char *event, xmlNode * msg) + stonith_enabled_s = crm_element_value(stonith_enabled_xml, XML_NVPAIR_ATTR_VALUE); + } + +- watchdog_device_update(local_cib); +- + if (stonith_enabled_s && crm_is_true(stonith_enabled_s) == FALSE) { + crm_trace("Ignoring cib updates while stonith is disabled"); + stonith_enabled_saved = FALSE; +- return; + + } else if (stonith_enabled_saved == FALSE) { + crm_info("Updating stonith device and topology lists now that stonith is enabled"); + stonith_enabled_saved = TRUE; +- fencing_topology_init(); +- cib_devices_update(); ++ need_full_refresh = TRUE; + + } else { +- update_fencing_topology(event, msg); +- update_cib_stonith_devices(event, msg); ++ if (timeout_ms_saved != stonith_watchdog_timeout_ms) { ++ need_full_refresh = TRUE; ++ } else { ++ update_fencing_topology(event, msg); ++ update_cib_stonith_devices(event, msg); ++ watchdog_device_update(); ++ } ++ } ++ ++ if (need_full_refresh) { ++ fencing_topology_init(); ++ cib_devices_update(); ++ watchdog_device_update(); + } + } + +@@ -1162,10 +1220,11 @@ init_cib_cache_cb(xmlNode * msg, int call_id, int rc, xmlNode * output, void *us + local_cib = copy_xml(output); + + crm_peer_caches_refresh(local_cib); ++ update_stonith_watchdog_timeout_ms(local_cib); + + fencing_topology_init(); +- watchdog_device_update(local_cib); + cib_devices_update(); ++ watchdog_device_update(); + } + + static void +diff --git a/daemons/fenced/pacemaker-fenced.h b/daemons/fenced/pacemaker-fenced.h +index d330fda4d..14e085e98 100644 +--- a/daemons/fenced/pacemaker-fenced.h ++++ b/daemons/fenced/pacemaker-fenced.h +@@ -260,14 +260,15 @@ bool fencing_peer_active(crm_node_t *peer); + + int stonith_manual_ack(xmlNode * msg, remote_fencing_op_t * op); + +-gboolean string_in_list(GListPtr list, const char *item); +- + gboolean node_has_attr(const char *node, const char *name, const char *value); + ++gboolean node_does_watchdog_fencing(const char *node); ++ + extern char *stonith_our_uname; + extern gboolean stand_alone; + extern GHashTable *device_list; + extern GHashTable *topology; + extern long stonith_watchdog_timeout_ms; ++extern GList *stonith_watchdog_targets; + + extern GHashTable *stonith_remote_op_list; +diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h +index 8bcb544d8..f222edba3 100644 +--- a/include/crm/fencing/internal.h ++++ b/include/crm/fencing/internal.h +@@ -164,7 +164,10 @@ void stonith__device_parameter_flags(uint32_t *device_flags, + # define STONITH_OP_LEVEL_ADD "st_level_add" + # define STONITH_OP_LEVEL_DEL "st_level_remove" + +-# define STONITH_WATCHDOG_AGENT "#watchdog" ++# define STONITH_WATCHDOG_AGENT "fence_watchdog" ++/* Don't change 2 below as it would break rolling upgrade */ ++# define STONITH_WATCHDOG_AGENT_INTERNAL "#watchdog" ++# define STONITH_WATCHDOG_ID "watchdog" + + # ifdef HAVE_STONITH_STONITH_H + // utilities from st_lha.c +@@ -211,4 +214,7 @@ stonith__event_state_pending(stonith_history_t *history, void *user_data) + bool stonith__event_state_eq(stonith_history_t *history, void *user_data); + bool stonith__event_state_neq(stonith_history_t *history, void *user_data); + ++gboolean stonith__watchdog_fencing_enabled_for_node(const char *node); ++gboolean stonith__watchdog_fencing_enabled_for_node_api(stonith_t *st, const char *node); ++ + #endif +diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c +index e285f51e2..0ff98157b 100644 +--- a/lib/fencing/st_client.c ++++ b/lib/fencing/st_client.c +@@ -195,6 +195,67 @@ stonith_get_namespace(const char *agent, const char *namespace_s) + return st_namespace_invalid; + } + ++gboolean ++stonith__watchdog_fencing_enabled_for_node_api(stonith_t *st, const char *node) ++{ ++ gboolean rv = FALSE; ++ stonith_t *stonith_api = st?st:stonith_api_new(); ++ char *list = NULL; ++ ++ if(stonith_api) { ++ if (stonith_api->state == stonith_disconnected) { ++ int rc = stonith_api->cmds->connect(stonith_api, "stonith-api", NULL); ++ ++ if (rc != pcmk_ok) { ++ crm_err("Failed connecting to Stonith-API for watchdog-fencing-query."); ++ } ++ } ++ ++ if (stonith_api->state != stonith_disconnected) { ++ /* caveat!!! ++ * this might fail when when stonithd is just updating the device-list ++ * probably something we should fix as well for other api-calls */ ++ int rc = stonith_api->cmds->list(stonith_api, st_opt_sync_call, STONITH_WATCHDOG_ID, &list, 0); ++ if ((rc != pcmk_ok) || (list == NULL)) { ++ /* due to the race described above it can happen that ++ * we drop in here - so as not to make remote nodes ++ * panic on that answer ++ */ ++ crm_warn("watchdog-fencing-query failed"); ++ } else if (list[0] == '\0') { ++ crm_warn("watchdog-fencing-query returned an empty list - any node"); ++ rv = TRUE; ++ } else { ++ GList *targets = stonith__parse_targets(list); ++ rv = pcmk__strcase_in_list(targets, node); ++ g_list_free_full(targets, free); ++ } ++ free(list); ++ if (!st) { ++ /* if we're provided the api we still might have done the ++ * connection - but let's assume the caller won't bother ++ */ ++ stonith_api->cmds->disconnect(stonith_api); ++ } ++ } ++ ++ if (!st) { ++ stonith_api_delete(stonith_api); ++ } ++ } else { ++ crm_err("Stonith-API for watchdog-fencing-query couldn't be created."); ++ } ++ crm_trace("Pacemaker assumes node %s %sto do watchdog-fencing.", ++ node, rv?"":"not "); ++ return rv; ++} ++ ++gboolean ++stonith__watchdog_fencing_enabled_for_node(const char *node) ++{ ++ return stonith__watchdog_fencing_enabled_for_node_api(NULL, node); ++} ++ + static void + log_action(stonith_action_t *action, pid_t pid) + { +diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c +index 87d050ed1..bf4bceb42 100644 +--- a/lib/lrmd/lrmd_client.c ++++ b/lib/lrmd/lrmd_client.c +@@ -34,6 +34,7 @@ + #include + + #include ++#include + + #ifdef HAVE_GNUTLS_GNUTLS_H + # undef KEYFILE +@@ -934,7 +935,10 @@ lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash) + crm_xml_add(data, F_LRMD_ORIGIN, __func__); + + value = g_hash_table_lookup(hash, "stonith-watchdog-timeout"); +- crm_xml_add(data, F_LRMD_WATCHDOG, value); ++ if ((value) && ++ (stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) { ++ crm_xml_add(data, F_LRMD_WATCHDOG, value); ++ } + + rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0, + (native->type == pcmk__client_ipc)); +diff --git a/rpm/pacemaker.spec.in b/rpm/pacemaker.spec.in +index 79e78ede9..f58357a77 100644 +--- a/rpm/pacemaker.spec.in ++++ b/rpm/pacemaker.spec.in +@@ -718,5 +718,6 @@ exit 0 + %{_sbindir}/crm_attribute + %{_sbindir}/crm_master + %{_sbindir}/fence_legacy ++%{_sbindir}/fence_watchdog + + %doc %{_mandir}/man7/pacemaker-controld.* +@@ -744,6 +744,7 @@ exit 0 + %doc %{_mandir}/man8/crm_attribute.* + %doc %{_mandir}/man8/crm_master.* + %doc %{_mandir}/man8/fence_legacy.* ++%doc %{_mandir}/man8/fence_watchdog.* + %doc %{_mandir}/man8/pacemakerd.* + + %doc %{_datadir}/pacemaker/alerts +@@ -822,6 +824,7 @@ exit 0 + %exclude %{_mandir}/man8/crm_attribute.* + %exclude %{_mandir}/man8/crm_master.* + %exclude %{_mandir}/man8/fence_legacy.* ++%exclude %{_mandir}/man8/fence_watchdog.* + %exclude %{_mandir}/man8/pacemakerd.* + %exclude %{_mandir}/man8/pacemaker-remoted.* + +-- +2.27.0 + + +From 53dd360f096e5f005e3221e8d44d82d3654b5172 Mon Sep 17 00:00:00 2001 +From: Klaus Wenninger +Date: Wed, 4 Aug 2021 15:57:23 +0200 +Subject: [PATCH 3/3] Fix: watchdog-fencing: Silence warning without node + restriction + +--- + lib/fencing/st_client.c | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c +index 0ff98157b..14fa7b2a6 100644 +--- a/lib/fencing/st_client.c ++++ b/lib/fencing/st_client.c +@@ -223,7 +223,6 @@ stonith__watchdog_fencing_enabled_for_node_api(stonith_t *st, const char *node) + */ + crm_warn("watchdog-fencing-query failed"); + } else if (list[0] == '\0') { +- crm_warn("watchdog-fencing-query returned an empty list - any node"); + rv = TRUE; + } else { + GList *targets = stonith__parse_targets(list); +-- +2.27.0 + +--- a/include/crm/common/strings_internal.h ++++ b/include/crm/common/strings_internal.h +@@ -42,6 +42,7 @@ + + int pcmk__parse_ll_range(const char *srcstring, long long *start, long long *end); + gboolean pcmk__str_in_list(GList *lst, const gchar *s); ++gboolean pcmk__strcase_in_list(GList *lst, const gchar *s); + + bool pcmk__strcase_any_of(const char *s, ...) G_GNUC_NULL_TERMINATED; + bool pcmk__str_any_of(const char *s, ...) G_GNUC_NULL_TERMINATED; +--- a/lib/common/strings.c ++++ b/lib/common/strings.c +@@ -804,6 +804,20 @@ + return g_list_find_custom(lst, s, (GCompareFunc) strcmp) != NULL; + } + ++gboolean ++pcmk__strcase_in_list(GList *lst, const gchar *s) ++{ ++ if (lst == NULL) { ++ return FALSE; ++ } ++ ++ if (strcmp(lst->data, "*") == 0 && lst->next == NULL) { ++ return TRUE; ++ } ++ ++ return g_list_find_custom(lst, s, (GCompareFunc) strcasecmp) != NULL; ++} ++ + static bool + str_any_of(bool casei, const char *s, va_list args) + { diff --git a/SOURCES/045-controller-attribute.patch b/SOURCES/045-controller-attribute.patch new file mode 100644 index 0000000..0271906 --- /dev/null +++ b/SOURCES/045-controller-attribute.patch @@ -0,0 +1,122 @@ +From ee7eba6a7a05bdf0a12d60ebabb334d8ee021101 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 9 Aug 2021 14:48:57 -0500 +Subject: [PATCH] Fix: controller: ensure lost node's transient attributes are + cleared without DC + +Previously, peer_update_callback() cleared a lost node's transient attributes +if either the local node is DC, or there is no DC. + +However, that left the possibility of the DC being lost at the same time as +another node -- the local node would still have fsa_our_dc set while processing +the leave notifications, so no node would clear the attributes for the non-DC +node. + +Now, the controller has its own CPG configuration change callback, which sets a +global boolean before calling the usual one, so that peer_update_callback() can +know when the DC has been lost. +--- + daemons/controld/controld_callbacks.c | 4 ++- + daemons/controld/controld_corosync.c | 57 ++++++++++++++++++++++++++++++++++- + 2 files changed, 59 insertions(+), 2 deletions(-) + +diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c +index af24856..e564b3d 100644 +--- a/daemons/controld/controld_callbacks.c ++++ b/daemons/controld/controld_callbacks.c +@@ -99,6 +99,8 @@ node_alive(const crm_node_t *node) + + #define state_text(state) ((state)? (const char *)(state) : "in unknown state") + ++bool controld_dc_left = false; ++ + void + peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data) + { +@@ -217,7 +219,7 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d + cib_scope_local); + } + +- } else if (AM_I_DC || (fsa_our_dc == NULL)) { ++ } else if (AM_I_DC || controld_dc_left || (fsa_our_dc == NULL)) { + /* This only needs to be done once, so normally the DC should do + * it. However if there is no DC, every node must do it, since + * there is no other way to ensure some one node does it. +diff --git a/daemons/controld/controld_corosync.c b/daemons/controld/controld_corosync.c +index db99630..c5ab658 100644 +--- a/daemons/controld/controld_corosync.c ++++ b/daemons/controld/controld_corosync.c +@@ -87,6 +87,61 @@ crmd_cs_destroy(gpointer user_data) + } + } + ++extern bool controld_dc_left; ++ ++/*! ++ * \brief Handle a Corosync notification of a CPG configuration change ++ * ++ * \param[in] handle CPG connection ++ * \param[in] cpg_name CPG group name ++ * \param[in] member_list List of current CPG members ++ * \param[in] member_list_entries Number of entries in \p member_list ++ * \param[in] left_list List of CPG members that left ++ * \param[in] left_list_entries Number of entries in \p left_list ++ * \param[in] joined_list List of CPG members that joined ++ * \param[in] joined_list_entries Number of entries in \p joined_list ++ */ ++static void ++cpg_membership_callback(cpg_handle_t handle, const struct cpg_name *cpg_name, ++ const struct cpg_address *member_list, ++ size_t member_list_entries, ++ const struct cpg_address *left_list, ++ size_t left_list_entries, ++ const struct cpg_address *joined_list, ++ size_t joined_list_entries) ++{ ++ /* When nodes leave CPG, the DC clears their transient node attributes. ++ * ++ * However if there is no DC, or the DC is among the nodes that left, each ++ * remaining node needs to do the clearing, to ensure it gets done. ++ * Otherwise, the attributes would persist when the nodes rejoin, which ++ * could have serious consequences for unfencing, agents that use attributes ++ * for internal logic, etc. ++ * ++ * Here, we set a global boolean if the DC is among the nodes that left, for ++ * use by the peer callback. ++ */ ++ if (fsa_our_dc != NULL) { ++ crm_node_t *peer = crm_find_peer(0, fsa_our_dc); ++ ++ if (peer != NULL) { ++ for (int i = 0; i < left_list_entries; ++i) { ++ if (left_list[i].nodeid == peer->id) { ++ controld_dc_left = true; ++ break; ++ } ++ } ++ } ++ } ++ ++ // Process the change normally, which will call the peer callback as needed ++ pcmk_cpg_membership(handle, cpg_name, member_list, member_list_entries, ++ left_list, left_list_entries, ++ joined_list, joined_list_entries); ++ ++ controld_dc_left = false; ++} ++ + extern gboolean crm_connect_corosync(crm_cluster_t * cluster); + + gboolean +@@ -95,7 +150,7 @@ crm_connect_corosync(crm_cluster_t * cluster) + if (is_corosync_cluster()) { + crm_set_status_callback(&peer_update_callback); + cluster->cpg.cpg_deliver_fn = crmd_cs_dispatch; +- cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership; ++ cluster->cpg.cpg_confchg_fn = cpg_membership_callback; + cluster->destroy = crmd_cs_destroy; + + if (crm_cluster_connect(cluster)) { +-- +1.8.3.1 + diff --git a/SPECS/pacemaker.spec b/SPECS/pacemaker.spec index 05ca75a..961fb71 100644 --- a/SPECS/pacemaker.spec +++ b/SPECS/pacemaker.spec @@ -226,7 +226,7 @@ Name: pacemaker Summary: Scalable High-Availability cluster resource manager Version: %{pcmkversion} -Release: %{pcmk_release}%{?dist}.1 +Release: %{pcmk_release}%{?dist}.3 %if %{defined _unitdir} License: GPLv2+ and LGPLv2+ %else @@ -289,6 +289,10 @@ Patch38: 038-feature-set.patch Patch39: 039-crm_mon.patch Patch40: 040-crm_mon-shutdown.patch Patch41: 041-crm_mon-shutdown.patch +Patch42: 042-unfencing-loop.patch +Patch43: 043-retry-metadata.patch +Patch44: 044-sbd.patch +Patch45: 045-controller-attribute.patch # downstream-only commits Patch100: 100-default-to-syncing-with-sbd.patch @@ -811,6 +815,7 @@ exit 0 %{_sbindir}/crm_attribute %{_sbindir}/crm_master +%{_sbindir}/fence_watchdog %doc %{_mandir}/man7/pacemaker-controld.* %doc %{_mandir}/man7/pacemaker-schedulerd.* @@ -819,6 +824,7 @@ exit 0 %doc %{_mandir}/man7/ocf_pacemaker_remote.* %doc %{_mandir}/man8/crm_attribute.* %doc %{_mandir}/man8/crm_master.* +%doc %{_mandir}/man8/fence_watchdog.* %doc %{_mandir}/man8/pacemakerd.* %doc %{_datadir}/pacemaker/alerts @@ -893,6 +899,7 @@ exit 0 %doc %{_mandir}/man8/* %exclude %{_mandir}/man8/crm_attribute.* %exclude %{_mandir}/man8/crm_master.* +%exclude %{_mandir}/man8/fence_watchdog.* %exclude %{_mandir}/man8/pacemakerd.* %exclude %{_mandir}/man8/pacemaker-remoted.* @@ -986,6 +993,18 @@ exit 0 %license %{nagios_name}-%{nagios_hash}/COPYING %changelog +* Mon Aug 9 2021 Klaus Wenninger - 2.0.5-9.3 +- retry fence-agent metadata +- assure transient attributes of lost node are cleared +- added configurable watchdog-fencing feature +- Resolves: rhbz1992014 +- Resolves: rhbz1989622 +- Resolves: rhbz1993891 + +* Thu Jun 24 2021 Ken Gaillot - 2.0.5-9.2 +- Avoid remote node unfencing loop +- Resolves: rhbz1972273 + * Mon Apr 19 2021 Ken Gaillot - 2.0.5-9.1 - Fix regression in crm_mon during cluster shutdown that affects ocf:heartbeat:pgsql agent - Resolves: rhbz1951098