Blame SOURCES/002-status-deletion.patch

308170
From 9e4addbcb67ea8e36ba853f1e401d8a6cb6a0aa3 Mon Sep 17 00:00:00 2001
308170
From: Ken Gaillot <kgaillot@redhat.com>
308170
Date: Fri, 20 Dec 2019 11:34:06 -0600
308170
Subject: [PATCH 1/8] Refactor: scheduler: reduce code duplication when
308170
 displaying resources
308170
308170
Refactor native_output_string() to use GString, for readability and
308170
maintainability. Refactor common_print() to use it, to reduce duplication and
308170
ensure displays are consistent.
308170
308170
This makes a couple small changes in how things are shown:
308170
308170
* If pe_print_dev is enabled (a debugging flag not actually used by anything),
308170
  the additional resource fields are shown with the resource flags rather than
308170
  their own parenthesized list.
308170
308170
* The new output model is now consistent with the legacy print model in
308170
  displaying resource flags with commas (not spaces) between them.
308170
---
308170
 include/crm/pengine/common.h |  24 +--
308170
 lib/pengine/native.c         | 410 +++++++++++++++++--------------------------
308170
 2 files changed, 168 insertions(+), 266 deletions(-)
308170
308170
diff --git a/include/crm/pengine/common.h b/include/crm/pengine/common.h
308170
index e497f9c..48c2b66 100644
308170
--- a/include/crm/pengine/common.h
308170
+++ b/include/crm/pengine/common.h
308170
@@ -1,22 +1,12 @@
308170
-/* 
308170
- * Copyright 2004-2018 the Pacemaker project contributors
308170
+/*
308170
+ * Copyright 2004-2019 the Pacemaker project contributors
308170
  *
308170
  * The version control history for this file may have further details.
308170
- * 
308170
- * This program is free software; you can redistribute it and/or
308170
- * modify it under the terms of the GNU Lesser General Public
308170
- * License as published by the Free Software Foundation; either
308170
- * version 2 of the License, or (at your option) any later version.
308170
- * 
308170
- * This software is distributed in the hope that it will be useful,
308170
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
308170
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
308170
- * General Public License for more details.
308170
- * 
308170
- * You should have received a copy of the GNU Lesser General Public
308170
- * License along with this library; if not, write to the Free Software
308170
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
308170
+ *
308170
+ * This source code is licensed under the GNU Lesser General Public License
308170
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
308170
  */
308170
+
308170
 #ifndef PE_COMMON__H
308170
 #  define PE_COMMON__H
308170
 
308170
@@ -104,7 +94,7 @@ enum pe_print_options {
308170
     pe_print_html           = 0x0002,
308170
     pe_print_ncurses        = 0x0004,
308170
     pe_print_printf         = 0x0008,
308170
-    pe_print_dev            = 0x0010,
308170
+    pe_print_dev            = 0x0010, // Debugging (@COMPAT probably not useful)
308170
     pe_print_details        = 0x0020,
308170
     pe_print_max_details    = 0x0040,
308170
     pe_print_rsconly        = 0x0080,
308170
diff --git a/lib/pengine/native.c b/lib/pengine/native.c
308170
index fdb98e0..8fd98bc 100644
308170
--- a/lib/pengine/native.c
308170
+++ b/lib/pengine/native.c
308170
@@ -1,5 +1,5 @@
308170
 /*
308170
- * Copyright 2004-2019 the Pacemaker project contributors
308170
+ * Copyright 2004-2020 the Pacemaker project contributors
308170
  *
308170
  * The version control history for this file may have further details.
308170
  *
308170
@@ -490,165 +490,172 @@ native_print_xml(resource_t * rsc, const char *pre_text, long options, void *pri
308170
     }
308170
 }
308170
 
308170
-/* making this inline rather than a macro prevents a coverity "unreachable"
308170
- * warning on the first usage
308170
- */
308170
-static inline const char *
308170
-comma_if(int i)
308170
+// Append a flag to resource description string's flags list
308170
+static bool
308170
+add_output_flag(GString *s, const char *flag_desc, bool have_flags)
308170
 {
308170
-    return i? ", " : "";
308170
+    g_string_append(s, (have_flags? ", " : " ("));
308170
+    g_string_append(s, flag_desc);
308170
+    return true;
308170
 }
308170
 
308170
-static char *
308170
-flags_string(pe_resource_t *rsc, pe_node_t *node, long options,
308170
-             const char *target_role)
308170
+// Append a node name to resource description string's node list
308170
+static bool
308170
+add_output_node(GString *s, const char *node, bool have_nodes)
308170
 {
308170
-    char *flags[6] = { NULL, };
308170
-    char *result = NULL;
308170
-    int ndx = 0;
308170
+    g_string_append(s, (have_nodes? " " : " [ "));
308170
+    g_string_append(s, node);
308170
+    return true;
308170
+}
308170
+
308170
+/*!
308170
+ * \internal
308170
+ * \brief Create a string description of a resource
308170
+ *
308170
+ * \param[in] rsc          Resource to describe
308170
+ * \param[in] name         Desired identifier for the resource
308170
+ * \param[in] node         If not NULL, node that resource is "on"
308170
+ * \param[in] options      Bitmask of pe_print_*
308170
+ * \param[in] target_role  Resource's target role
308170
+ * \param[in] show_nodes   Whether to display nodes when multiply active
308170
+ *
308170
+ * \return Newly allocated string description of resource
308170
+ * \note Caller must free the result with g_free().
308170
+ */
308170
+static gchar *
308170
+native_output_string(pe_resource_t *rsc, const char *name, pe_node_t *node,
308170
+                     long options, const char *target_role, bool show_nodes)
308170
+{
308170
+    const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
308170
+    const char *provider = NULL;
308170
+    const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
308170
+    char *retval = NULL;
308170
+    GString *outstr = NULL;
308170
+    bool have_flags = false;
308170
+
308170
+    CRM_CHECK(name != NULL, name = "unknown");
308170
+    CRM_CHECK(kind != NULL, kind = "unknown");
308170
+    CRM_CHECK(class != NULL, class = "unknown");
308170
+
308170
+    if (is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
308170
+        provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
308170
+    }
308170
 
308170
-    if (node && node->details->online == FALSE && node->details->unclean) {
308170
-        flags[ndx++] = strdup("UNCLEAN");
308170
+    if (is_set(options, pe_print_rsconly)
308170
+        || pcmk__list_of_multiple(rsc->running_on)) {
308170
+        node = NULL;
308170
     }
308170
 
308170
+    // We need a string of at least this size
308170
+    outstr = g_string_sized_new(strlen(name) + strlen(class) + strlen(kind)
308170
+                                + (provider? (strlen(provider) + 2) : 0)
308170
+                                + (node? strlen(node->details->uname) + 1 : 0)
308170
+                                + 11);
308170
+
308170
+    // Resource name and agent
308170
+    g_string_printf(outstr, "%s\t(%s%s%s:%s):\t", name, class,
308170
+                    /* @COMPAT This should be a single ':' (see CLBZ#5395) but
308170
+                     * to avoid breaking anything relying on it, we're keeping
308170
+                     * it like this until the next minor version bump.
308170
+                     */
308170
+                    (provider? "::" : ""), (provider? provider : ""), kind);
308170
+
308170
+    // State on node
308170
+    if (is_set(rsc->flags, pe_rsc_orphan)) {
308170
+        g_string_append(outstr, " ORPHANED");
308170
+    }
308170
+    if (is_set(rsc->flags, pe_rsc_failed)) {
308170
+        enum rsc_role_e role = native_displayable_role(rsc);
308170
+
308170
+        if (role > RSC_ROLE_SLAVE) {
308170
+            g_string_append_printf(outstr, " FAILED %s", role2text(role));
308170
+        } else {
308170
+            g_string_append(outstr, " FAILED");
308170
+        }
308170
+    } else {
308170
+        g_string_append(outstr, native_displayable_state(rsc, options));
308170
+    }
308170
+    if (node) {
308170
+        g_string_append_printf(outstr, " %s", node->details->uname);
308170
+    }
308170
+
308170
+    // Flags, as: (<flag> [...])
308170
+    if (node && !(node->details->online) && node->details->unclean) {
308170
+        have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
308170
+    }
308170
     if (is_set(options, pe_print_pending)) {
308170
         const char *pending_task = native_pending_task(rsc);
308170
 
308170
         if (pending_task) {
308170
-            flags[ndx++] = strdup(pending_task);
308170
+            have_flags = add_output_flag(outstr, pending_task, have_flags);
308170
         }
308170
     }
308170
-
308170
     if (target_role) {
308170
         enum rsc_role_e target_role_e = text2role(target_role);
308170
 
308170
-        /* Ignore target role Started, as it is the default anyways
308170
-         * (and would also allow a Master to be Master).
308170
-         * Show if target role limits our abilities. */
308170
+        /* Only show target role if it limits our abilities (i.e. ignore
308170
+         * Started, as it is the default anyways, and doesn't prevent the
308170
+         * resource from becoming Master).
308170
+         */
308170
         if (target_role_e == RSC_ROLE_STOPPED) {
308170
-            flags[ndx++] = strdup("disabled");
308170
+            have_flags = add_output_flag(outstr, "disabled", have_flags);
308170
 
308170
         } else if (is_set(uber_parent(rsc)->flags, pe_rsc_promotable)
308170
                    && target_role_e == RSC_ROLE_SLAVE) {
308170
-            flags[ndx++] = crm_strdup_printf("target-role:%s", target_role);
308170
+            have_flags = add_output_flag(outstr, "target-role:", have_flags);
308170
+            g_string_append(outstr, target_role);
308170
         }
308170
     }
308170
-
308170
     if (is_set(rsc->flags, pe_rsc_block)) {
308170
-        flags[ndx++] = strdup("blocked");
308170
-
308170
+        have_flags = add_output_flag(outstr, "blocked", have_flags);
308170
     } else if (is_not_set(rsc->flags, pe_rsc_managed)) {
308170
-        flags[ndx++] = strdup("unmanaged");
308170
+        have_flags = add_output_flag(outstr, "unmanaged", have_flags);
308170
     }
308170
-
308170
     if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
308170
-        flags[ndx++] = strdup("failure ignored");
308170
+        have_flags = add_output_flag(outstr, "failure ignored", have_flags);
308170
     }
308170
-
308170
-    if (ndx > 0) {
308170
-        char *total = g_strjoinv(" ", flags);
308170
-
308170
-        result = crm_strdup_printf(" (%s)", total);
308170
-        g_free(total);
308170
-    }
308170
-
308170
-    while (--ndx >= 0) {
308170
-        free(flags[ndx]);
308170
-    }
308170
-    return result;
308170
-}
308170
-
308170
-static char *
308170
-native_output_string(resource_t *rsc, const char *name, node_t *node, long options,
308170
-                     const char *target_role) {
308170
-    const char *desc = NULL;
308170
-    const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
308170
-    const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
308170
-    enum rsc_role_e role = native_displayable_role(rsc);
308170
-
308170
-    char *retval = NULL;
308170
-
308170
-    char *unames = NULL;
308170
-    char *provider = NULL;
308170
-    const char *orphan = NULL;
308170
-    char *role_s = NULL;
308170
-    char *node_s = NULL;
308170
-    char *print_dev_s = NULL;
308170
-    char *flags_s = NULL;
308170
-
308170
-    CRM_ASSERT(kind != NULL);
308170
-
308170
-    if (is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
308170
-        provider = crm_strdup_printf("::%s", crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER));
308170
+    if (is_set(options, pe_print_dev)) {
308170
+        if (is_set(options, pe_rsc_provisional)) {
308170
+            have_flags = add_output_flag(outstr, "provisional", have_flags);
308170
+        }
308170
+        if (is_not_set(options, pe_rsc_runnable)) {
308170
+            have_flags = add_output_flag(outstr, "non-startable", have_flags);
308170
+        }
308170
+        have_flags = add_output_flag(outstr, "variant:", have_flags);
308170
+        g_string_append_printf(outstr, "%s priority:%f",
308170
+                                       crm_element_name(rsc->xml),
308170
+                                       (double) (rsc->priority));
308170
     }
308170
-
308170
-    if (is_set(rsc->flags, pe_rsc_orphan)) {
308170
-        orphan = " ORPHANED";
308170
+    if (have_flags) {
308170
+        g_string_append(outstr, ")");
308170
     }
308170
 
308170
-    if (role > RSC_ROLE_SLAVE && is_set(rsc->flags, pe_rsc_failed)) {
308170
-        role_s = crm_strdup_printf(" FAILED %s", role2text(role));
308170
-    } else if (is_set(rsc->flags, pe_rsc_failed)) {
308170
-        role_s = crm_strdup_printf(" FAILED");
308170
-    } else {
308170
-        role_s = crm_strdup_printf(" %s", native_displayable_state(rsc, options));
308170
-    }
308170
+    // User-supplied description
308170
+    if (is_set(options, pe_print_rsconly)
308170
+        || pcmk__list_of_multiple(rsc->running_on)) {
308170
+        const char *desc = crm_element_value(rsc->xml, XML_ATTR_DESC);
308170
 
308170
-    if (node) {
308170
-        node_s = crm_strdup_printf(" %s", node->details->uname);
308170
+        if (desc) {
308170
+            g_string_append_printf(outstr, " %s", desc);
308170
+        }
308170
     }
308170
 
308170
-    if (is_set(options, pe_print_rsconly) || g_list_length(rsc->running_on) > 1) {
308170
-        desc = crm_element_value(rsc->xml, XML_ATTR_DESC);
308170
-    }
308170
+    if (show_nodes && is_not_set(options, pe_print_rsconly)
308170
+        && pcmk__list_of_multiple(rsc->running_on)) {
308170
+        bool have_nodes = false;
308170
 
308170
-    if (is_not_set(options, pe_print_rsconly) && g_list_length(rsc->running_on) > 1) {
308170
-        GListPtr gIter = rsc->running_on;
308170
-        gchar **arr = calloc(g_list_length(rsc->running_on)+1, sizeof(gchar *));
308170
-        int i = 0;
308170
-        char *total = NULL;
308170
+        for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
308170
+            pe_node_t *n = (pe_node_t *) iter->data;
308170
 
308170
-        for (; gIter != NULL; gIter = gIter->next) {
308170
-            node_t *n = (node_t *) gIter->data;
308170
-            arr[i] = (gchar *) strdup(n->details->uname);
308170
-            i++;
308170
+            have_nodes = add_output_node(outstr, n->details->uname, have_nodes);
308170
+        }
308170
+        if (have_nodes) {
308170
+            g_string_append(outstr, " ]");
308170
         }
308170
-
308170
-        total = g_strjoinv(" ", arr);
308170
-        unames = crm_strdup_printf(" [ %s ]", total);
308170
-
308170
-        g_free(total);
308170
-        g_strfreev(arr);
308170
     }
308170
 
308170
-    if (is_set(options, pe_print_dev)) {
308170
-        print_dev_s = crm_strdup_printf(" (%s%svariant=%s, priority=%f)",
308170
-                                        is_set(rsc->flags, pe_rsc_provisional) ? "provisional, " : "",
308170
-                                        is_set(rsc->flags, pe_rsc_runnable) ? "" : "non-startable, ",
308170
-                                        crm_element_name(rsc->xml), (double)rsc->priority);
308170
-    }
308170
-
308170
-    flags_s = flags_string(rsc, node, options, target_role);
308170
-
308170
-    retval = crm_strdup_printf("%s\t(%s%s:%s):\t%s%s%s%s%s%s%s%s",
308170
-                               name, class,
308170
-                               provider ? provider : "",
308170
-                               kind,
308170
-                               orphan ? orphan : "",
308170
-                               role_s,
308170
-                               node_s ? node_s : "",
308170
-                               print_dev_s ? print_dev_s : "",
308170
-                               flags_s ? flags_s : "",
308170
-                               desc ? " " : "", desc ? desc : "",
308170
-                               unames ? unames : "");
308170
-
308170
-    free(provider);
308170
-    free(role_s);
308170
-    free(node_s);
308170
-    free(unames);
308170
-    free(print_dev_s);
308170
-    free(flags_s);
308170
-
308170
+    retval = outstr->str;
308170
+    g_string_free(outstr, FALSE);
308170
     return retval;
308170
 }
308170
 
308170
@@ -656,7 +663,6 @@ void
308170
 pe__common_output_html(pcmk__output_t *out, resource_t * rsc,
308170
                        const char *name, node_t *node, long options)
308170
 {
308170
-    char *s = NULL;
308170
     const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
308170
     const char *target_role = NULL;
308170
 
308170
@@ -675,10 +681,6 @@ pe__common_output_html(pcmk__output_t *out, resource_t * rsc,
308170
         target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
308170
     }
308170
 
308170
-    if ((options & pe_print_rsconly) || g_list_length(rsc->running_on) > 1) {
308170
-        node = NULL;
308170
-    }
308170
-
308170
     if (is_not_set(rsc->flags, pe_rsc_managed)) {
308170
         cl = "rsc-managed";
308170
 
308170
@@ -698,10 +700,14 @@ pe__common_output_html(pcmk__output_t *out, resource_t * rsc,
308170
         cl = "rsc-ok";
308170
     }
308170
 
308170
-    s = native_output_string(rsc, name, node, options, target_role);
308170
-    list_node = pcmk__output_create_html_node(out, "li", NULL, NULL, NULL);
308170
-    pcmk_create_html_node(list_node, "span", NULL, cl, s);
308170
-    free(s);
308170
+    {
308170
+        gchar *s = native_output_string(rsc, name, node, options, target_role,
308170
+                                        true);
308170
+
308170
+        list_node = pcmk__output_create_html_node(out, "li", NULL, NULL, NULL);
308170
+        pcmk_create_html_node(list_node, "span", NULL, cl, s);
308170
+        g_free(s);
308170
+    }
308170
 
308170
     if (is_set(options, pe_print_details)) {
308170
         GHashTableIter iter;
308170
@@ -744,7 +750,6 @@ void
308170
 pe__common_output_text(pcmk__output_t *out, resource_t * rsc,
308170
                        const char *name, node_t *node, long options)
308170
 {
308170
-    char *s = NULL;
308170
     const char *target_role = NULL;
308170
 
308170
     CRM_ASSERT(rsc->variant == pe_native);
308170
@@ -758,13 +763,13 @@ pe__common_output_text(pcmk__output_t *out, resource_t * rsc,
308170
         target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
308170
     }
308170
 
308170
-    if (is_set(options, pe_print_rsconly) || g_list_length(rsc->running_on) > 1) {
308170
-        node = NULL;
308170
-    }
308170
+    {
308170
+        gchar *s = native_output_string(rsc, name, node, options, target_role,
308170
+                                        true);
308170
 
308170
-    s = native_output_string(rsc, name, node, options, target_role);
308170
-    out->list_item(out, NULL, "%s", s);
308170
-    free(s);
308170
+        out->list_item(out, NULL, "%s", s);
308170
+        g_free(s);
308170
+    }
308170
 
308170
     if (is_set(options, pe_print_details)) {
308170
         GHashTableIter iter;
308170
@@ -806,22 +811,14 @@ pe__common_output_text(pcmk__output_t *out, resource_t * rsc,
308170
 void
308170
 common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *node, long options, void *print_data)
308170
 {
308170
-    const char *desc = NULL;
308170
-    const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
308170
-    const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
308170
     const char *target_role = NULL;
308170
-    enum rsc_role_e role = native_displayable_role(rsc);
308170
-
308170
-    int offset = 0;
308170
-    int flagOffset = 0;
308170
-    char buffer[LINE_MAX];
308170
-    char flagBuffer[LINE_MAX];
308170
 
308170
     CRM_ASSERT(rsc->variant == pe_native);
308170
-    CRM_ASSERT(kind != NULL);
308170
 
308170
     if (rsc->meta) {
308170
-        const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);
308170
+        const char *is_internal = g_hash_table_lookup(rsc->meta,
308170
+                                                      XML_RSC_ATTR_INTERNAL_RSC);
308170
+
308170
         if (crm_is_true(is_internal) && is_not_set(options, pe_print_implicit)) {
308170
             crm_trace("skipping print of internal resource %s", rsc->id);
308170
             return;
308170
@@ -829,17 +826,13 @@ common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *n
308170
         target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
308170
     }
308170
 
308170
-    if (pre_text == NULL && (options & pe_print_printf)) {
308170
-        pre_text = " ";
308170
-    }
308170
-
308170
     if (options & pe_print_xml) {
308170
         native_print_xml(rsc, pre_text, options, print_data);
308170
         return;
308170
     }
308170
 
308170
-    if ((options & pe_print_rsconly) || g_list_length(rsc->running_on) > 1) {
308170
-        node = NULL;
308170
+    if ((pre_text == NULL) && (options & pe_print_printf)) {
308170
+        pre_text = " ";
308170
     }
308170
 
308170
     if (options & pe_print_html) {
308170
@@ -849,10 +842,10 @@ common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *n
308170
         } else if (is_set(rsc->flags, pe_rsc_failed)) {
308170
             status_print("<font color=\"red\">");
308170
 
308170
-        } else if (rsc->variant == pe_native && (rsc->running_on == NULL)) {
308170
+        } else if (rsc->running_on == NULL) {
308170
             status_print("<font color=\"red\">");
308170
 
308170
-        } else if (g_list_length(rsc->running_on) > 1) {
308170
+        } else if (pcmk__list_of_multiple(rsc->running_on)) {
308170
             status_print("<font color=\"orange\">");
308170
 
308170
         } else if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
308170
@@ -863,106 +856,29 @@ common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *n
308170
         }
308170
     }
308170
 
308170
-    if(pre_text) {
308170
-        offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", pre_text);
308170
-    }
308170
-    offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", name);
308170
-    offset += snprintf(buffer + offset, LINE_MAX - offset, "\t(%s", class);
308170
-    if (is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
308170
-        const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
308170
-        offset += snprintf(buffer + offset, LINE_MAX - offset, "::%s", prov);
308170
-    }
308170
-    offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s):\t", kind);
308170
-    if(is_set(rsc->flags, pe_rsc_orphan)) {
308170
-        offset += snprintf(buffer + offset, LINE_MAX - offset, " ORPHANED ");
308170
-    }
308170
-    if(role > RSC_ROLE_SLAVE && is_set(rsc->flags, pe_rsc_failed)) {
308170
-        offset += snprintf(buffer + offset, LINE_MAX - offset, "FAILED %s", role2text(role));
308170
-    } else if(is_set(rsc->flags, pe_rsc_failed)) {
308170
-        offset += snprintf(buffer + offset, LINE_MAX - offset, "FAILED");
308170
-    } else {
308170
-        const char *rsc_state = native_displayable_state(rsc, options);
308170
-
308170
-        offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", rsc_state);
308170
-    }
308170
-
308170
-    if(node) {
308170
-        offset += snprintf(buffer + offset, LINE_MAX - offset, " %s", node->details->uname);
308170
-
308170
-        if (node->details->online == FALSE && node->details->unclean) {
308170
-            flagOffset += snprintf(flagBuffer + flagOffset, LINE_MAX - flagOffset,
308170
-                                   "%sUNCLEAN", comma_if(flagOffset));
308170
-        }
308170
-    }
308170
-
308170
-    if (options & pe_print_pending) {
308170
-        const char *pending_task = native_pending_task(rsc);
308170
-
308170
-        if (pending_task) {
308170
-            flagOffset += snprintf(flagBuffer + flagOffset, LINE_MAX - flagOffset,
308170
-                                   "%s%s", comma_if(flagOffset), pending_task);
308170
-        }
308170
-    }
308170
-
308170
-    if (target_role) {
308170
-        enum rsc_role_e target_role_e = text2role(target_role);
308170
-
308170
-        /* Ignore target role Started, as it is the default anyways
308170
-         * (and would also allow a Master to be Master).
308170
-         * Show if target role limits our abilities. */
308170
-        if (target_role_e == RSC_ROLE_STOPPED) {
308170
-            flagOffset += snprintf(flagBuffer + flagOffset, LINE_MAX - flagOffset,
308170
-                                   "%sdisabled", comma_if(flagOffset));
308170
-
308170
-        } else if (is_set(uber_parent(rsc)->flags, pe_rsc_promotable)
308170
-                   && target_role_e == RSC_ROLE_SLAVE) {
308170
-            flagOffset += snprintf(flagBuffer + flagOffset, LINE_MAX - flagOffset,
308170
-                                   "%starget-role:%s", comma_if(flagOffset), target_role);
308170
-        }
308170
-    }
308170
-
308170
-    if (is_set(rsc->flags, pe_rsc_block)) {
308170
-        flagOffset += snprintf(flagBuffer + flagOffset, LINE_MAX - flagOffset,
308170
-                               "%sblocked", comma_if(flagOffset));
308170
-
308170
-    } else if (is_not_set(rsc->flags, pe_rsc_managed)) {
308170
-        flagOffset += snprintf(flagBuffer + flagOffset, LINE_MAX - flagOffset,
308170
-                               "%sunmanaged", comma_if(flagOffset));
308170
-    }
308170
-
308170
-    if(is_set(rsc->flags, pe_rsc_failure_ignored)) {
308170
-        flagOffset += snprintf(flagBuffer + flagOffset, LINE_MAX - flagOffset,
308170
-                               "%sfailure ignored", comma_if(flagOffset));
308170
-    }
308170
-
308170
-    if ((options & pe_print_rsconly) || g_list_length(rsc->running_on) > 1) {
308170
-        desc = crm_element_value(rsc->xml, XML_ATTR_DESC);
308170
-    }
308170
-
308170
-    CRM_LOG_ASSERT(offset > 0);
308170
-    if(flagOffset > 0) {
308170
-        status_print("%s (%s)%s%s", buffer, flagBuffer, desc?" ":"", desc?desc:"");
308170
-    } else {
308170
-        status_print("%s%s%s", buffer, desc?" ":"", desc?desc:"");
308170
+    {
308170
+        gchar *resource_s = native_output_string(rsc, name, node, options,
308170
+                                                 target_role, false);
308170
+        status_print("%s%s", (pre_text? pre_text : ""), resource_s);
308170
+        g_free(resource_s);
308170
     }
308170
 
308170
 #if CURSES_ENABLED
308170
-    if ((options & pe_print_rsconly) || g_list_length(rsc->running_on) > 1) {
308170
-        /* Done */
308170
-
308170
-    } else if (options & pe_print_ncurses) {
308170
+    if (is_set(options, pe_print_ncurses)
308170
+        && is_not_set(options, pe_print_rsconly)
308170
+        && !pcmk__list_of_multiple(rsc->running_on)) {
308170
         /* coverity[negative_returns] False positive */
308170
         move(-1, 0);
308170
     }
308170
 #endif
308170
 
308170
-    if (options & pe_print_html) {
308170
+    if (is_set(options, pe_print_html)) {
308170
         status_print(" </font> ");
308170
     }
308170
 
308170
-    if ((options & pe_print_rsconly)) {
308170
+    if (is_not_set(options, pe_print_rsconly)
308170
+        && pcmk__list_of_multiple(rsc->running_on)) {
308170
 
308170
-    } else if (g_list_length(rsc->running_on) > 1) {
308170
         GListPtr gIter = rsc->running_on;
308170
         int counter = 0;
308170
 
308170
@@ -1025,10 +941,6 @@ common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *n
308170
         GHashTableIter iter;
308170
         node_t *n = NULL;
308170
 
308170
-        status_print("%s\t(%s%svariant=%s, priority=%f)", pre_text,
308170
-                     is_set(rsc->flags, pe_rsc_provisional) ? "provisional, " : "",
308170
-                     is_set(rsc->flags, pe_rsc_runnable) ? "" : "non-startable, ",
308170
-                     crm_element_name(rsc->xml), (double)rsc->priority);
308170
         status_print("%s\tAllowed Nodes", pre_text);
308170
         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
308170
         while (g_hash_table_iter_next(&iter, NULL, (void **)&n)) {
308170
-- 
308170
1.8.3.1
308170
308170
308170
From 41e911be8ea9151b3f0758c2c22c0e69b8b78d93 Mon Sep 17 00:00:00 2001
308170
From: Ken Gaillot <kgaillot@redhat.com>
308170
Date: Thu, 19 Dec 2019 17:18:41 -0600
308170
Subject: [PATCH 2/8] Log: scheduler: drop redundant trace messages
308170
308170
We logged "applying placement constraints" three times.
308170
---
308170
 lib/pacemaker/pcmk_sched_allocate.c | 17 ++++-------------
308170
 1 file changed, 4 insertions(+), 13 deletions(-)
308170
308170
diff --git a/lib/pacemaker/pcmk_sched_allocate.c b/lib/pacemaker/pcmk_sched_allocate.c
308170
index ca43c71..dde8b69 100644
308170
--- a/lib/pacemaker/pcmk_sched_allocate.c
308170
+++ b/lib/pacemaker/pcmk_sched_allocate.c
308170
@@ -623,21 +623,15 @@ check_actions(pe_working_set_t * data_set)
308170
     }
308170
 }
308170
 
308170
-static gboolean
308170
+static void
308170
 apply_placement_constraints(pe_working_set_t * data_set)
308170
 {
308170
-    GListPtr gIter = NULL;
308170
-
308170
-    crm_trace("Applying constraints...");
308170
-
308170
-    for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
308170
+    for (GList *gIter = data_set->placement_constraints;
308170
+         gIter != NULL; gIter = gIter->next) {
308170
         pe__location_t *cons = gIter->data;
308170
 
308170
         cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
308170
     }
308170
-
308170
-    return TRUE;
308170
-
308170
 }
308170
 
308170
 static gboolean
308170
@@ -994,10 +988,7 @@ stage2(pe_working_set_t * data_set)
308170
 {
308170
     GListPtr gIter = NULL;
308170
 
308170
-    crm_trace("Applying placement constraints");
308170
-
308170
-    gIter = data_set->nodes;
308170
-    for (; gIter != NULL; gIter = gIter->next) {
308170
+    for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
308170
         node_t *node = (node_t *) gIter->data;
308170
 
308170
         if (node == NULL) {
308170
-- 
308170
1.8.3.1
308170
308170
308170
From 7fe136e19b5018d609beb8bad4e34234739572c9 Mon Sep 17 00:00:00 2001
308170
From: Ken Gaillot <kgaillot@redhat.com>
308170
Date: Sat, 7 Dec 2019 12:13:11 -0600
308170
Subject: [PATCH 3/8] Refactor: libcrmcommon: convenience functions for list
308170
 length comparisons
308170
308170
... for efficiency and readability
308170
---
308170
 include/crm/common/internal.h | 14 ++++++++++++++
308170
 1 file changed, 14 insertions(+)
308170
308170
diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h
308170
index da2c7d7..484c836 100644
308170
--- a/include/crm/common/internal.h
308170
+++ b/include/crm/common/internal.h
308170
@@ -126,6 +126,20 @@ crm_getpid_s()
308170
     return crm_strdup_printf("%lu", (unsigned long) getpid());
308170
 }
308170
 
308170
+// More efficient than g_list_length(list) == 1
308170
+static inline bool
308170
+pcmk__list_of_1(GList *list)
308170
+{
308170
+    return list && (list->next == NULL);
308170
+}
308170
+
308170
+// More efficient than g_list_length(list) > 1
308170
+static inline bool
308170
+pcmk__list_of_multiple(GList *list)
308170
+{
308170
+    return list && (list->next != NULL);
308170
+}
308170
+
308170
 /* convenience functions for failure-related node attributes */
308170
 
308170
 #define CRM_FAIL_COUNT_PREFIX   "fail-count"
308170
-- 
308170
1.8.3.1
308170
308170
308170
From 9ff4f6bca540576f0a3333c959e8014ed168353f Mon Sep 17 00:00:00 2001
308170
From: Ken Gaillot <kgaillot@redhat.com>
308170
Date: Mon, 16 Dec 2019 14:13:30 -0600
308170
Subject: [PATCH 4/8] Refactor: libcrmcommon: add convenience macros for
308170
 plurals
308170
308170
I've avoided making s_if_plural() an official API due to its hackiness, but
308170
it really is the best solution for now. Promote it to pcmk__plural_s(), along
308170
with a companion macro pcmk__plural_alt() for more complicated plurals.
308170
---
308170
 include/crm/common/internal.h | 23 +++++++++++++++++++++++
308170
 1 file changed, 23 insertions(+)
308170
308170
diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h
308170
index 484c836..ee560c9 100644
308170
--- a/include/crm/common/internal.h
308170
+++ b/include/crm/common/internal.h
308170
@@ -107,6 +107,29 @@ bool crm_compress_string(const char *data, int length, int max, char **result,
308170
                          unsigned int *result_len);
308170
 gint crm_alpha_sort(gconstpointer a, gconstpointer b);
308170
 
308170
+/* Correctly displaying singular or plural is complicated; consider "1 node has"
308170
+ * vs. "2 nodes have". A flexible solution is to pluralize entire strings, e.g.
308170
+ *
308170
+ * if (a == 1) {
308170
+ *     crm_info("singular message"):
308170
+ * } else {
308170
+ *     crm_info("plural message");
308170
+ * }
308170
+ *
308170
+ * though even that's not sufficient for all languages besides English (if we
308170
+ * ever desire to do translations of output and log messages). But the following
308170
+ * convenience macros are "good enough" and more concise for many cases.
308170
+ */
308170
+
308170
+/* Example:
308170
+ * crm_info("Found %d %s", nentries,
308170
+ *          pcmk__plural_alt(nentries, "entry", "entries"));
308170
+ */
308170
+#define pcmk__plural_alt(i, s1, s2) (((i) == 1)? (s1) : (s2))
308170
+
308170
+// Example: crm_info("Found %d node%s", nnodes, pcmk__plural_s(nnodes));
308170
+#define pcmk__plural_s(i) pcmk__plural_alt(i, "", "s")
308170
+
308170
 static inline char *
308170
 crm_concat(const char *prefix, const char *suffix, char join)
308170
 {
308170
-- 
308170
1.8.3.1
308170
308170
308170
From 0378db5030400202e59b2bae0dabd65d00a3e9c8 Mon Sep 17 00:00:00 2001
308170
From: Ken Gaillot <kgaillot@redhat.com>
308170
Date: Thu, 12 Dec 2019 20:50:50 -0600
308170
Subject: [PATCH 5/8] Log: controller: improve join messages
308170
308170
---
308170
 daemons/controld/controld_fsa.c     |  81 ++++----
308170
 daemons/controld/controld_join_dc.c | 383 +++++++++++++++++++++---------------
308170
 2 files changed, 268 insertions(+), 196 deletions(-)
308170
308170
diff --git a/daemons/controld/controld_fsa.c b/daemons/controld/controld_fsa.c
308170
index 6760224..b985fa9 100644
308170
--- a/daemons/controld/controld_fsa.c
308170
+++ b/daemons/controld/controld_fsa.c
308170
@@ -1,5 +1,5 @@
308170
 /*
308170
- * Copyright 2004-2019 the Pacemaker project contributors
308170
+ * Copyright 2004-2020 the Pacemaker project contributors
308170
  *
308170
  * The version control history for this file may have further details.
308170
  *
308170
@@ -460,12 +460,53 @@ log_fsa_input(fsa_data_t * stored_msg)
308170
     }
308170
 }
308170
 
308170
+static void
308170
+check_join_counts(fsa_data_t *msg_data)
308170
+{
308170
+    int count;
308170
+    guint npeers;
308170
+
308170
+    count = crmd_join_phase_count(crm_join_finalized);
308170
+    if (count > 0) {
308170
+        crm_err("%d cluster node%s failed to confirm join",
308170
+                count, pcmk__plural_s(count));
308170
+        crmd_join_phase_log(LOG_NOTICE);
308170
+        return;
308170
+    }
308170
+
308170
+    npeers = crm_active_peers();
308170
+    count = crmd_join_phase_count(crm_join_confirmed);
308170
+    if (count == npeers) {
308170
+        if (npeers == 1) {
308170
+            crm_debug("Sole active cluster node is fully joined");
308170
+        } else {
308170
+            crm_debug("All %d active cluster nodes are fully joined", count);
308170
+        }
308170
+
308170
+    } else if (count > npeers) {
308170
+        crm_err("New election needed because more nodes confirmed join "
308170
+                "than are in membership (%d > %u)", count, npeers);
308170
+        register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
308170
+
308170
+    } else if (saved_ccm_membership_id != crm_peer_seq) {
308170
+        crm_info("New join needed because membership changed (%llu -> %llu)",
308170
+                 saved_ccm_membership_id, crm_peer_seq);
308170
+        register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
308170
+
308170
+    } else {
308170
+        crm_warn("Only %d of %u active cluster nodes fully joined "
308170
+                 "(%d did not respond to offer)",
308170
+                 count, npeers, crmd_join_phase_count(crm_join_welcomed));
308170
+    }
308170
+}
308170
+
308170
 long long
308170
 do_state_transition(long long actions,
308170
                     enum crmd_fsa_state cur_state,
308170
                     enum crmd_fsa_state next_state, fsa_data_t * msg_data)
308170
 {
308170
     int level = LOG_INFO;
308170
+    int count = 0;
308170
     long long tmp = actions;
308170
     gboolean clear_recovery_bit = TRUE;
308170
 
308170
@@ -563,13 +604,14 @@ do_state_transition(long long actions,
308170
                 crm_warn("Progressed to state %s after %s",
308170
                          fsa_state2string(next_state), fsa_cause2string(cause));
308170
             }
308170
-            if (crmd_join_phase_count(crm_join_welcomed) > 0) {
308170
-                crm_warn("%u cluster nodes failed to respond"
308170
-                         " to the join offer.", crmd_join_phase_count(crm_join_welcomed));
308170
+            count = crmd_join_phase_count(crm_join_welcomed);
308170
+            if (count > 0) {
308170
+                crm_warn("%d cluster node%s failed to respond to join offer",
308170
+                         count, pcmk__plural_s(count));
308170
                 crmd_join_phase_log(LOG_NOTICE);
308170
 
308170
             } else {
308170
-                crm_debug("All %d cluster nodes responded to the join offer.",
308170
+                crm_debug("All cluster nodes (%d) responded to join offer",
308170
                           crmd_join_phase_count(crm_join_integrated));
308170
             }
308170
             break;
308170
@@ -581,34 +623,7 @@ do_state_transition(long long actions,
308170
                 crm_info("Progressed to state %s after %s",
308170
                          fsa_state2string(next_state), fsa_cause2string(cause));
308170
             }
308170
-
308170
-            if (crmd_join_phase_count(crm_join_finalized) > 0) {
308170
-                crm_err("%u cluster nodes failed to confirm their join.",
308170
-                        crmd_join_phase_count(crm_join_finalized));
308170
-                crmd_join_phase_log(LOG_NOTICE);
308170
-
308170
-            } else if (crmd_join_phase_count(crm_join_confirmed)
308170
-                       == crm_active_peers()) {
308170
-                crm_debug("All %u cluster nodes are"
308170
-                          " eligible to run resources.", crm_active_peers());
308170
-
308170
-            } else if (crmd_join_phase_count(crm_join_confirmed) > crm_active_peers()) {
308170
-                crm_err("We have more confirmed nodes than our membership does: %d vs. %d",
308170
-                        crmd_join_phase_count(crm_join_confirmed), crm_active_peers());
308170
-                register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
308170
-
308170
-            } else if (saved_ccm_membership_id != crm_peer_seq) {
308170
-                crm_info("Membership changed: %llu -> %llu - join restart",
308170
-                         saved_ccm_membership_id, crm_peer_seq);
308170
-                register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
308170
-
308170
-            } else {
308170
-                crm_warn("Only %u of %u cluster "
308170
-                         "nodes are eligible to run resources - continue %d",
308170
-                         crmd_join_phase_count(crm_join_confirmed),
308170
-                         crm_active_peers(), crmd_join_phase_count(crm_join_welcomed));
308170
-            }
308170
-/* 			initialize_join(FALSE); */
308170
+            check_join_counts(msg_data);
308170
             break;
308170
 
308170
         case S_STOPPING:
308170
diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c
308170
index 988aaa6..54324b2 100644
308170
--- a/daemons/controld/controld_join_dc.c
308170
+++ b/daemons/controld/controld_join_dc.c
308170
@@ -26,7 +26,11 @@ void finalize_join_for(gpointer key, gpointer value, gpointer user_data);
308170
 void finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
308170
 gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
308170
 
308170
+/* Numeric counter used to identify join rounds (an unsigned int would be
308170
+ * appropriate, except we get and set it in XML as int)
308170
+ */
308170
 static int current_join_id = 0;
308170
+
308170
 unsigned long long saved_ccm_membership_id = 0;
308170
 
308170
 void
308170
@@ -34,12 +38,7 @@ crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase
308170
 {
308170
     enum crm_join_phase last = 0;
308170
 
308170
-    if(node == NULL) {
308170
-        crm_err("Could not update join because node not specified"
308170
-                CRM_XS " join-%u source=%s phase=%s",
308170
-                current_join_id, source, crm_join_phase_str(phase));
308170
-        return;
308170
-    }
308170
+    CRM_CHECK(node != NULL, return);
308170
 
308170
     /* Remote nodes do not participate in joins */
308170
     if (is_set(node->flags, crm_remote_node)) {
308170
@@ -49,21 +48,23 @@ crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase
308170
     last = node->join;
308170
 
308170
     if(phase == last) {
308170
-        crm_trace("%s: Node %s[%u] - join-%u phase still %s",
308170
-                  source, node->uname, node->id, current_join_id,
308170
-                  crm_join_phase_str(last));
308170
+        crm_trace("Node %s join-%d phase is still %s "
308170
+                  CRM_XS " nodeid=%u source=%s",
308170
+                  node->uname, current_join_id, crm_join_phase_str(last),
308170
+                  node->id, source);
308170
 
308170
     } else if ((phase <= crm_join_none) || (phase == (last + 1))) {
308170
         node->join = phase;
308170
-        crm_info("%s: Node %s[%u] - join-%u phase %s -> %s",
308170
-                 source, node->uname, node->id, current_join_id,
308170
-                 crm_join_phase_str(last), crm_join_phase_str(phase));
308170
+        crm_trace("Node %s join-%d phase is now %s (was %s) "
308170
+                  CRM_XS " nodeid=%u source=%s",
308170
+                 node->uname, current_join_id, crm_join_phase_str(phase),
308170
+                 crm_join_phase_str(last), node->id, source);
308170
 
308170
     } else {
308170
-        crm_err("Could not update join for node %s because phase transition invalid "
308170
-                CRM_XS " join-%u source=%s node_id=%u last=%s new=%s",
308170
-                node->uname, current_join_id, source, node->id,
308170
-                crm_join_phase_str(last), crm_join_phase_str(phase));
308170
+        crm_warn("Rejecting join-%d phase update for node %s because "
308170
+                 "can't go from %s to %s " CRM_XS " nodeid=%u source=%s",
308170
+                 current_join_id, node->uname, crm_join_phase_str(last),
308170
+                 crm_join_phase_str(phase), node->id, source);
308170
     }
308170
 }
308170
 
308170
@@ -73,9 +74,7 @@ initialize_join(gboolean before)
308170
     GHashTableIter iter;
308170
     crm_node_t *peer = NULL;
308170
 
308170
-    /* clear out/reset a bunch of stuff */
308170
-    crm_debug("join-%d: Initializing join data (flag=%s)",
308170
-              current_join_id, before ? "true" : "false");
308170
+    crm_debug("Starting new join round join-%d", current_join_id);
308170
 
308170
     g_hash_table_iter_init(&iter, crm_peer_cache);
308170
     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
308170
@@ -128,7 +127,9 @@ join_make_offer(gpointer key, gpointer value, gpointer user_data)
308170
 
308170
     CRM_ASSERT(member != NULL);
308170
     if (crm_is_peer_active(member) == FALSE) {
308170
-        crm_info("Not making an offer to %s: not active (%s)", member->uname, member->state);
308170
+        crm_info("Not making join-%d offer to inactive node %s",
308170
+                 current_join_id,
308170
+                 (member->uname? member->uname : "with unknown name"));
308170
         if(member->expected == NULL && safe_str_eq(member->state, CRM_NODE_LOST)) {
308170
             /* You would think this unsafe, but in fact this plus an
308170
              * active resource is what causes it to be fenced.
308170
@@ -145,17 +146,21 @@ join_make_offer(gpointer key, gpointer value, gpointer user_data)
308170
     }
308170
 
308170
     if (member->uname == NULL) {
308170
-        crm_info("No recipient for welcome message.(Node uuid:%s)", member->uuid);
308170
+        crm_info("Not making join-%d offer to node uuid %s with unknown name",
308170
+                 current_join_id, member->uuid);
308170
         return;
308170
     }
308170
 
308170
     if (saved_ccm_membership_id != crm_peer_seq) {
308170
         saved_ccm_membership_id = crm_peer_seq;
308170
-        crm_info("Making join offers based on membership %llu", crm_peer_seq);
308170
+        crm_info("Making join-%d offers based on membership event %llu",
308170
+                 current_join_id, crm_peer_seq);
308170
     }
308170
 
308170
     if(user_data && member->join > crm_join_none) {
308170
-        crm_info("Skipping %s: already known %d", member->uname, member->join);
308170
+        crm_info("Not making join-%d offer to already known node %s (%s)",
308170
+                 current_join_id, member->uname,
308170
+                 crm_join_phase_str(member->join));
308170
         return;
308170
     }
308170
 
308170
@@ -166,14 +171,11 @@ join_make_offer(gpointer key, gpointer value, gpointer user_data)
308170
     // Advertise our feature set so the joining node can bail if not compatible
308170
     crm_xml_add(offer, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
308170
 
308170
-    /* send the welcome */
308170
-    crm_info("join-%d: Sending offer to %s", current_join_id, member->uname);
308170
-
308170
+    crm_info("Sending join-%d offer to %s", current_join_id, member->uname);
308170
     send_cluster_message(member, crm_msg_crmd, offer, TRUE);
308170
     free_xml(offer);
308170
 
308170
     crm_update_peer_join(__FUNCTION__, member, crm_join_welcomed);
308170
-    /* crm_update_peer_expected(__FUNCTION__, member, CRMD_JOINSTATE_PENDING); */
308170
 }
308170
 
308170
 /*	 A_DC_JOIN_OFFER_ALL	*/
308170
@@ -183,6 +185,8 @@ do_dc_join_offer_all(long long action,
308170
                      enum crmd_fsa_state cur_state,
308170
                      enum crmd_fsa_input current_input, fsa_data_t * msg_data)
308170
 {
308170
+    int count;
308170
+
308170
     /* Reset everyone's status back to down or in_ccm in the CIB.
308170
      * Any nodes that are active in the CIB but not in the cluster membership
308170
      * will be seen as offline by the scheduler anyway.
308170
@@ -197,9 +201,11 @@ do_dc_join_offer_all(long long action,
308170
     }
308170
     g_hash_table_foreach(crm_peer_cache, join_make_offer, NULL);
308170
 
308170
+    count = crmd_join_phase_count(crm_join_welcomed);
308170
+    crm_info("Waiting on join-%d requests from %d outstanding node%s",
308170
+             current_join_id, count, pcmk__plural_s(count));
308170
+
308170
     // Don't waste time by invoking the scheduler yet
308170
-    crm_info("join-%d: Waiting on %d outstanding join acks",
308170
-             current_join_id, crmd_join_phase_count(crm_join_welcomed));
308170
 }
308170
 
308170
 /*	 A_DC_JOIN_OFFER_ONE	*/
308170
@@ -211,50 +217,40 @@ do_dc_join_offer_one(long long action,
308170
 {
308170
     crm_node_t *member;
308170
     ha_msg_input_t *welcome = NULL;
308170
-
308170
-    const char *op = NULL;
308170
+    int count;
308170
     const char *join_to = NULL;
308170
 
308170
-    if (msg_data->data) {
308170
-        welcome = fsa_typed_data(fsa_dt_ha_msg);
308170
-
308170
-    } else {
308170
-        crm_info("An unknown node joined - (re-)offer to any unconfirmed nodes");
308170
+    if (msg_data->data == NULL) {
308170
+        crm_info("Making join-%d offers to any unconfirmed nodes "
308170
+                 "because an unknown node joined", current_join_id);
308170
         g_hash_table_foreach(crm_peer_cache, join_make_offer, &member);
308170
         check_join_state(cur_state, __FUNCTION__);
308170
         return;
308170
     }
308170
 
308170
+    welcome = fsa_typed_data(fsa_dt_ha_msg);
308170
     if (welcome == NULL) {
308170
-        crm_err("Attempt to send welcome message without a message to reply to!");
308170
+        // fsa_typed_data() already logged an error
308170
         return;
308170
     }
308170
 
308170
     join_to = crm_element_value(welcome->msg, F_CRM_HOST_FROM);
308170
     if (join_to == NULL) {
308170
-        crm_err("Attempt to send welcome message without a host to reply to!");
308170
+        crm_err("Can't make join-%d offer to unknown node", current_join_id);
308170
         return;
308170
     }
308170
-
308170
     member = crm_get_peer(0, join_to);
308170
-    op = crm_element_value(welcome->msg, F_CRM_TASK);
308170
-    if (join_to != NULL && (cur_state == S_INTEGRATION || cur_state == S_FINALIZE_JOIN)) {
308170
-        /* note: it _is_ possible that a node will have been
308170
-         *  sick or starting up when the original offer was made.
308170
-         *  however, it will either re-announce itself in due course
308170
-         *  _or_ we can re-store the original offer on the client.
308170
-         */
308170
-        crm_trace("(Re-)offering membership to %s...", join_to);
308170
-    }
308170
 
308170
-    crm_info("join-%d: Processing %s request from %s in state %s",
308170
-             current_join_id, op, join_to, fsa_state2string(cur_state));
308170
+    /* It is possible that a node will have been sick or starting up when the
308170
+     * original offer was made. However, it will either re-announce itself in
308170
+     * due course, or we can re-store the original offer on the client.
308170
+     */
308170
 
308170
     crm_update_peer_join(__FUNCTION__, member, crm_join_none);
308170
     join_make_offer(NULL, member, NULL);
308170
 
308170
-    /* always offer to the DC (ourselves)
308170
-     * this ensures the correct value for max_generation_from
308170
+    /* If the offer isn't to the local node, make an offer to the local node as
308170
+     * well, to ensure the correct value for max_generation_from.
308170
      */
308170
     if (strcmp(join_to, fsa_our_uname) != 0) {
308170
         member = crm_get_peer(0, fsa_our_uname);
308170
@@ -266,9 +262,11 @@ do_dc_join_offer_one(long long action,
308170
      */
308170
     abort_transition(INFINITY, tg_restart, "Node join", NULL);
308170
 
308170
+    count = crmd_join_phase_count(crm_join_welcomed);
308170
+    crm_info("Waiting on join-%d requests from %d outstanding node%s",
308170
+             current_join_id, count, pcmk__plural_s(count));
308170
+
308170
     // Don't waste time by invoking the scheduler yet
308170
-    crm_debug("Waiting on %d outstanding join acks for join-%d",
308170
-              crmd_join_phase_count(crm_join_welcomed), current_join_id);
308170
 }
308170
 
308170
 static int
308170
@@ -301,22 +299,31 @@ do_dc_join_filter_offer(long long action,
308170
 
308170
     int cmp = 0;
308170
     int join_id = -1;
308170
+    int count = 0;
308170
     gboolean ack_nack_bool = TRUE;
308170
-    const char *ack_nack = CRMD_JOINSTATE_MEMBER;
308170
     ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
308170
 
308170
     const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
308170
     const char *ref = crm_element_value(join_ack->msg, F_CRM_REFERENCE);
308170
     const char *join_version = crm_element_value(join_ack->msg,
308170
                                                  XML_ATTR_CRM_VERSION);
308170
+    crm_node_t *join_node = NULL;
308170
 
308170
-    crm_node_t *join_node = crm_get_peer(0, join_from);
308170
-
308170
-    crm_debug("Processing req from %s", join_from);
308170
+    if (join_from == NULL) {
308170
+        crm_err("Ignoring invalid join request without node name");
308170
+        return;
308170
+    }
308170
+    join_node = crm_get_peer(0, join_from);
308170
 
308170
-    generation = join_ack->xml;
308170
     crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id);
308170
+    if (join_id != current_join_id) {
308170
+        crm_debug("Ignoring join-%d request from %s because we are on join-%d",
308170
+                  join_id, join_from, current_join_id);
308170
+        check_join_state(cur_state, __FUNCTION__);
308170
+        return;
308170
+    }
308170
 
308170
+    generation = join_ack->xml;
308170
     if (max_generation_xml != NULL && generation != NULL) {
308170
         int lpc = 0;
308170
 
308170
@@ -331,68 +338,71 @@ do_dc_join_filter_offer(long long action,
308170
         }
308170
     }
308170
 
308170
-    if (join_id != current_join_id) {
308170
-        crm_debug("Invalid response from %s: join-%d vs. join-%d",
308170
-                  join_from, join_id, current_join_id);
308170
-        check_join_state(cur_state, __FUNCTION__);
308170
-        return;
308170
+    if (ref == NULL) {
308170
+        ref = "none"; // for logging only
308170
+    }
308170
 
308170
-    } else if (join_node == NULL || crm_is_peer_active(join_node) == FALSE) {
308170
-        crm_err("Node %s is not a member", join_from);
308170
+    if (crm_is_peer_active(join_node) == FALSE) {
308170
+        crm_err("Rejecting join-%d request from inactive node %s "
308170
+                CRM_XS " ref=%s", join_id, join_from, ref);
308170
         ack_nack_bool = FALSE;
308170
 
308170
     } else if (generation == NULL) {
308170
-        crm_err("Generation was NULL");
308170
+        crm_err("Rejecting invalid join-%d request from node %s "
308170
+                "missing CIB generation " CRM_XS " ref=%s",
308170
+                join_id, join_from, ref);
308170
         ack_nack_bool = FALSE;
308170
 
308170
     } else if ((join_version == NULL)
308170
                || !feature_set_compatible(CRM_FEATURE_SET, join_version)) {
308170
-        crm_err("Node %s feature set (%s) is incompatible with ours (%s)",
308170
-                join_from, (join_version? join_version : "pre-3.1.0"),
308170
-                CRM_FEATURE_SET);
308170
+        crm_err("Rejecting join-%d request from node %s because feature set %s"
308170
+                " is incompatible with ours (%s) " CRM_XS " ref=%s",
308170
+                join_id, join_from, (join_version? join_version : "pre-3.1.0"),
308170
+                CRM_FEATURE_SET, ref);
308170
         ack_nack_bool = FALSE;
308170
 
308170
     } else if (max_generation_xml == NULL) {
308170
+        crm_debug("Accepting join-%d request from %s "
308170
+                  "(with first CIB generation) " CRM_XS " ref=%s",
308170
+                  join_id, join_from, ref);
308170
         max_generation_xml = copy_xml(generation);
308170
         max_generation_from = strdup(join_from);
308170
 
308170
     } else if (cmp < 0 || (cmp == 0 && safe_str_eq(join_from, fsa_our_uname))) {
308170
-        crm_debug("%s has a better generation number than"
308170
-                  " the current max %s", join_from, max_generation_from);
308170
-        if (max_generation_xml) {
308170
-            crm_log_xml_debug(max_generation_xml, "Max generation");
308170
-        }
308170
-        crm_log_xml_debug(generation, "Their generation");
308170
+        crm_debug("Accepting join-%d request from %s (with better "
308170
+                  "CIB generation than current best from %s) " CRM_XS " ref=%s",
308170
+                  join_id, join_from, max_generation_from, ref);
308170
+        crm_log_xml_debug(max_generation_xml, "Old max generation");
308170
+        crm_log_xml_debug(generation, "New max generation");
308170
 
308170
         free(max_generation_from);
308170
         free_xml(max_generation_xml);
308170
 
308170
         max_generation_from = strdup(join_from);
308170
         max_generation_xml = copy_xml(join_ack->xml);
308170
+
308170
+    } else {
308170
+        crm_debug("Accepting join-%d request from %s " CRM_XS " ref=%s",
308170
+                  join_id, join_from, ref);
308170
     }
308170
 
308170
     if (ack_nack_bool == FALSE) {
308170
-        /* NACK this client */
308170
-        ack_nack = CRMD_JOINSTATE_NACK;
308170
         crm_update_peer_join(__FUNCTION__, join_node, crm_join_nack);
308170
-        crm_err("Rejecting cluster join request from %s " CRM_XS
308170
-                " NACK join-%d ref=%s", join_from, join_id, ref);
308170
-
308170
+        crm_update_peer_expected(__FUNCTION__, join_node, CRMD_JOINSTATE_NACK);
308170
     } else {
308170
-        crm_debug("join-%d: Welcoming node %s (ref %s)", join_id, join_from, ref);
308170
         crm_update_peer_join(__FUNCTION__, join_node, crm_join_integrated);
308170
+        crm_update_peer_expected(__FUNCTION__, join_node, CRMD_JOINSTATE_MEMBER);
308170
     }
308170
 
308170
-    crm_update_peer_expected(__FUNCTION__, join_node, ack_nack);
308170
-
308170
-    crm_debug("%u nodes have been integrated into join-%d",
308170
-              crmd_join_phase_count(crm_join_integrated), join_id);
308170
-
308170
+    count = crmd_join_phase_count(crm_join_integrated);
308170
+    crm_debug("%d node%s currently integrated in join-%d",
308170
+              count, pcmk__plural_s(count), join_id);
308170
 
308170
     if (check_join_state(cur_state, __FUNCTION__) == FALSE) {
308170
         // Don't waste time by invoking the scheduler yet
308170
-        crm_debug("join-%d: Still waiting on %d outstanding offers",
308170
-                  join_id, crmd_join_phase_count(crm_join_welcomed));
308170
+        count = crmd_join_phase_count(crm_join_welcomed);
308170
+        crm_debug("Waiting on join-%d requests from %d outstanding node%s",
308170
+                  join_id, count, pcmk__plural_s(count));
308170
     }
308170
 }
308170
 
308170
@@ -405,21 +415,24 @@ do_dc_join_finalize(long long action,
308170
 {
308170
     char *sync_from = NULL;
308170
     int rc = pcmk_ok;
308170
+    int count_welcomed = crmd_join_phase_count(crm_join_welcomed);
308170
+    int count_integrated = crmd_join_phase_count(crm_join_integrated);
308170
 
308170
     /* This we can do straight away and avoid clients timing us out
308170
      *  while we compute the latest CIB
308170
      */
308170
-    crm_debug("Finalizing join-%d for %d clients",
308170
-              current_join_id, crmd_join_phase_count(crm_join_integrated));
308170
-
308170
-    crmd_join_phase_log(LOG_INFO);
308170
-    if (crmd_join_phase_count(crm_join_welcomed) != 0) {
308170
-        crm_info("Waiting for %d more nodes", crmd_join_phase_count(crm_join_welcomed));
308170
+    if (count_welcomed != 0) {
308170
+        crm_debug("Waiting on join-%d requests from %d outstanding node%s "
308170
+                  "before finalizing join", current_join_id, count_welcomed,
308170
+                  pcmk__plural_s(count_welcomed));
308170
+        crmd_join_phase_log(LOG_DEBUG);
308170
         /* crmd_fsa_stall(FALSE); Needed? */
308170
         return;
308170
 
308170
-    } else if (crmd_join_phase_count(crm_join_integrated) == 0) {
308170
-        /* Nothing to do */
308170
+    } else if (count_integrated == 0) {
308170
+        crm_debug("Finalization not needed for join-%d at the current time",
308170
+                  current_join_id);
308170
+        crmd_join_phase_log(LOG_DEBUG);
308170
         check_join_state(fsa_state, __FUNCTION__);
308170
         return;
308170
     }
308170
@@ -430,8 +443,9 @@ do_dc_join_finalize(long long action,
308170
     }
308170
 
308170
     if (is_set(fsa_input_register, R_IN_TRANSITION)) {
308170
-        crm_warn("Delaying response to cluster join offer while transition in progress "
308170
-                 CRM_XS " join-%d", current_join_id);
308170
+        crm_warn("Delaying join-%d finalization while transition in progress",
308170
+                 current_join_id);
308170
+        crmd_join_phase_log(LOG_DEBUG);
308170
         crmd_fsa_stall(FALSE);
308170
         return;
308170
     }
308170
@@ -440,18 +454,20 @@ do_dc_join_finalize(long long action,
308170
         /* ask for the agreed best CIB */
308170
         sync_from = strdup(max_generation_from);
308170
         set_bit(fsa_input_register, R_CIB_ASKED);
308170
-        crm_notice("Syncing the Cluster Information Base from %s to rest of cluster "
308170
-                   CRM_XS " join-%d", sync_from, current_join_id);
308170
-        crm_log_xml_notice(max_generation_xml, "Requested version");
308170
+        crm_notice("Finalizing join-%d for %d node%s (sync'ing CIB from %s)",
308170
+                   current_join_id, count_integrated,
308170
+                   pcmk__plural_s(count_integrated), sync_from);
308170
+        crm_log_xml_notice(max_generation_xml, "Requested CIB version");
308170
 
308170
     } else {
308170
         /* Send _our_ CIB out to everyone */
308170
         sync_from = strdup(fsa_our_uname);
308170
-        crm_info("join-%d: Syncing our CIB to the rest of the cluster",
308170
-                 current_join_id);
308170
-        crm_log_xml_debug(max_generation_xml, "Requested version");
308170
+        crm_debug("Finalizing join-%d for %d node%s (sync'ing from local CIB)",
308170
+                  current_join_id, count_integrated,
308170
+                  pcmk__plural_s(count_integrated));
308170
+        crm_log_xml_debug(max_generation_xml, "Requested CIB version");
308170
     }
308170
-
308170
+    crmd_join_phase_log(LOG_DEBUG);
308170
 
308170
     rc = fsa_cib_conn->cmds->sync_from(fsa_cib_conn, sync_from, NULL, cib_quorum_override);
308170
     fsa_register_cib_callback(rc, FALSE, sync_from, finalize_sync_callback);
308170
@@ -463,26 +479,33 @@ finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, voi
308170
     CRM_LOG_ASSERT(-EPERM != rc);
308170
     clear_bit(fsa_input_register, R_CIB_ASKED);
308170
     if (rc != pcmk_ok) {
308170
-        do_crm_log((rc == -pcmk_err_old_data ? LOG_WARNING : LOG_ERR),
308170
-                   "Sync from %s failed: %s", (char *)user_data, pcmk_strerror(rc));
308170
+        do_crm_log(((rc == -pcmk_err_old_data)? LOG_WARNING : LOG_ERR),
308170
+                   "Could not sync CIB from %s in join-%d: %s",
308170
+                   (char *) user_data, current_join_id, pcmk_strerror(rc));
308170
 
308170
         /* restart the whole join process */
308170
         register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION_DC, NULL, NULL, __FUNCTION__);
308170
 
308170
-    } else if (AM_I_DC && fsa_state == S_FINALIZE_JOIN) {
308170
+    } else if (!AM_I_DC) {
308170
+        crm_debug("Sync'ed CIB for join-%d but no longer DC", current_join_id);
308170
+
308170
+    } else if (fsa_state != S_FINALIZE_JOIN) {
308170
+        crm_debug("Sync'ed CIB for join-%d but no longer in S_FINALIZE_JOIN (%s)",
308170
+                  current_join_id, fsa_state2string(fsa_state));
308170
+
308170
+    } else {
308170
         set_bit(fsa_input_register, R_HAVE_CIB);
308170
         clear_bit(fsa_input_register, R_CIB_ASKED);
308170
 
308170
         /* make sure dc_uuid is re-set to us */
308170
         if (check_join_state(fsa_state, __FUNCTION__) == FALSE) {
308170
-            crm_debug("Notifying %d clients of join-%d results",
308170
-                      crmd_join_phase_count(crm_join_integrated), current_join_id);
308170
+            int count_integrated = crmd_join_phase_count(crm_join_integrated);
308170
+
308170
+            crm_debug("Notifying %d node%s of join-%d results",
308170
+                      count_integrated, pcmk__plural_s(count_integrated),
308170
+                      current_join_id);
308170
             g_hash_table_foreach(crm_peer_cache, finalize_join_for, NULL);
308170
         }
308170
-
308170
-    } else {
308170
-        crm_debug("No longer the DC in S_FINALIZE_JOIN: %s in %s",
308170
-                  AM_I_DC ? "DC" : "controller", fsa_state2string(fsa_state));
308170
     }
308170
 }
308170
 
308170
@@ -492,11 +515,14 @@ join_update_complete_callback(xmlNode * msg, int call_id, int rc, xmlNode * outp
308170
     fsa_data_t *msg_data = NULL;
308170
 
308170
     if (rc == pcmk_ok) {
308170
-        crm_debug("Join update %d complete", call_id);
308170
+        crm_debug("join-%d node history update (via CIB call %d) complete",
308170
+                  current_join_id, call_id);
308170
         check_join_state(fsa_state, __FUNCTION__);
308170
 
308170
     } else {
308170
-        crm_err("Join update %d failed", call_id);
308170
+        crm_err("join-%d node history update (via CIB call %d) failed: %s "
308170
+                "(next transition may determine resource status incorrectly)",
308170
+                current_join_id, call_id, pcmk_strerror(rc));
308170
         crm_log_xml_debug(msg, "failed");
308170
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
308170
     }
308170
@@ -515,61 +541,75 @@ do_dc_join_ack(long long action,
308170
 
308170
     const char *op = crm_element_value(join_ack->msg, F_CRM_TASK);
308170
     const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
308170
-    crm_node_t *peer = crm_get_peer(0, join_from);
308170
+    crm_node_t *peer = NULL;
308170
 
308170
-    if (safe_str_neq(op, CRM_OP_JOIN_CONFIRM) || peer == NULL) {
308170
-        crm_debug("Ignoring op=%s message from %s", op, join_from);
308170
+    // Sanity checks
308170
+    if (join_from == NULL) {
308170
+        crm_warn("Ignoring message received without node identification");
308170
+        return;
308170
+    }
308170
+    if (op == NULL) {
308170
+        crm_warn("Ignoring message received from %s without task", join_from);
308170
         return;
308170
     }
308170
 
308170
-    crm_trace("Processing ack from %s", join_from);
308170
-    crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id);
308170
+    if (strcmp(op, CRM_OP_JOIN_CONFIRM)) {
308170
+        crm_debug("Ignoring '%s' message from %s while waiting for '%s'",
308170
+                  op, join_from, CRM_OP_JOIN_CONFIRM);
308170
+        return;
308170
+    }
308170
 
308170
+    if (crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id) != 0) {
308170
+        crm_warn("Ignoring join confirmation from %s without valid join ID",
308170
+                 join_from);
308170
+        return;
308170
+    }
308170
+
308170
+    peer = crm_get_peer(0, join_from);
308170
     if (peer->join != crm_join_finalized) {
308170
-        crm_info("Join not in progress: ignoring join-%d from %s (phase = %d)",
308170
-                 join_id, join_from, peer->join);
308170
+        crm_info("Ignoring out-of-sequence join-%d confirmation from %s "
308170
+                 "(currently %s not %s)",
308170
+                 join_id, join_from, crm_join_phase_str(peer->join),
308170
+                 crm_join_phase_str(crm_join_finalized));
308170
         return;
308170
+    }
308170
 
308170
-    } else if (join_id != current_join_id) {
308170
-        crm_err("Invalid response from %s: join-%d vs. join-%d",
308170
-                join_from, join_id, current_join_id);
308170
+    if (join_id != current_join_id) {
308170
+        crm_err("Rejecting join-%d confirmation from %s "
308170
+                "because currently on join-%d",
308170
+                join_id, join_from, current_join_id);
308170
         crm_update_peer_join(__FUNCTION__, peer, crm_join_nack);
308170
         return;
308170
     }
308170
 
308170
     crm_update_peer_join(__FUNCTION__, peer, crm_join_confirmed);
308170
 
308170
-    crm_info("join-%d: Updating node state to %s for %s",
308170
-             join_id, CRMD_JOINSTATE_MEMBER, join_from);
308170
-
308170
-    /* update CIB with the current LRM status from the node
308170
-     * We don't need to notify the TE of these updates, a transition will
308170
-     *   be started in due time
308170
+    /* Update CIB with node's current executor state. A new transition will be
308170
+     * triggered later, when the CIB notifies us of the change.
308170
      */
308170
     erase_status_tag(join_from, XML_CIB_TAG_LRM, cib_scope_local);
308170
-
308170
     if (safe_str_eq(join_from, fsa_our_uname)) {
308170
         xmlNode *now_dc_lrmd_state = do_lrm_query(TRUE, fsa_our_uname);
308170
 
308170
         if (now_dc_lrmd_state != NULL) {
308170
-            crm_debug("Local executor state updated from query");
308170
             fsa_cib_update(XML_CIB_TAG_STATUS, now_dc_lrmd_state,
308170
                 cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL);
308170
             free_xml(now_dc_lrmd_state);
308170
+            crm_debug("Updating local node history for join-%d "
308170
+                      "from query result (via CIB call %d)", join_id, call_id);
308170
         } else {
308170
-            crm_warn("Local executor state updated from join acknowledgement because query failed");
308170
             fsa_cib_update(XML_CIB_TAG_STATUS, join_ack->xml,
308170
                 cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL);
308170
+            crm_warn("Updating local node history from join-%d confirmation "
308170
+                     "because query failed (via CIB call %d)", join_id, call_id);
308170
         }
308170
     } else {
308170
-        crm_debug("Executor state for %s updated from join acknowledgement",
308170
-                  join_from);
308170
         fsa_cib_update(XML_CIB_TAG_STATUS, join_ack->xml,
308170
            cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL);
308170
+        crm_debug("Updating node history for %s from join-%d confirmation "
308170
+                  "(via CIB call %d)", join_from, join_id, call_id);
308170
     }
308170
-
308170
     fsa_register_cib_callback(call_id, FALSE, NULL, join_update_complete_callback);
308170
-    crm_debug("join-%d: Registered callback for CIB status update %d", join_id, call_id);
308170
 }
308170
 
308170
 void
308170
@@ -581,17 +621,16 @@ finalize_join_for(gpointer key, gpointer value, gpointer user_data)
308170
     const char *join_to = join_node->uname;
308170
 
308170
     if(join_node->join != crm_join_integrated) {
308170
-        crm_trace("Skipping %s in state %d", join_to, join_node->join);
308170
+        crm_trace("Not updating non-integrated node %s (%s) for join-%d",
308170
+                  join_to, crm_join_phase_str(join_node->join),
308170
+                  current_join_id);
308170
         return;
308170
     }
308170
 
308170
-    /* make sure a node entry exists for the new node */
308170
-    crm_trace("Creating node entry for %s", join_to);
308170
-
308170
+    crm_trace("Updating node state for %s", join_to);
308170
     tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE);
308170
     set_uuid(tmp1, XML_ATTR_UUID, join_node);
308170
     crm_xml_add(tmp1, XML_ATTR_UNAME, join_to);
308170
-
308170
     fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1);
308170
     free_xml(tmp1);
308170
 
308170
@@ -610,11 +649,10 @@ finalize_join_for(gpointer key, gpointer value, gpointer user_data)
308170
         return;
308170
     }
308170
 
308170
-    /* send the ack/nack to the node */
308170
-    acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to);
308170
-
308170
-    crm_debug("join-%d: ACK'ing join request from %s",
308170
+    // Acknowledge node's join request
308170
+    crm_debug("Acknowledging join-%d request from %s",
308170
               current_join_id, join_to);
308170
+    acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to);
308170
     crm_xml_add(acknak, CRM_OP_JOIN_ACKNAK, XML_BOOLEAN_TRUE);
308170
     crm_update_peer_join(__FUNCTION__, join_node, crm_join_finalized);
308170
     crm_update_peer_expected(__FUNCTION__, join_node, CRMD_JOINSTATE_MEMBER);
308170
@@ -629,11 +667,11 @@ check_join_state(enum crmd_fsa_state cur_state, const char *source)
308170
 {
308170
     static unsigned long long highest_seq = 0;
308170
 
308170
-    crm_debug("Invoked by %s in state: %s", source, fsa_state2string(cur_state));
308170
-
308170
     if (saved_ccm_membership_id != crm_peer_seq) {
308170
-        crm_debug("%s: Membership changed since join started: %llu -> %llu (%llu)",
308170
-                  source, saved_ccm_membership_id, crm_peer_seq, highest_seq);
308170
+        crm_debug("join-%d: Membership changed from %llu to %llu "
308170
+                  CRM_XS " highest=%llu state=%s for=%s",
308170
+                  current_join_id, saved_ccm_membership_id, crm_peer_seq, highest_seq,
308170
+                  fsa_state2string(cur_state), source);
308170
         if(highest_seq < crm_peer_seq) {
308170
             /* Don't spam the FSA with duplicates */
308170
             highest_seq = crm_peer_seq;
308170
@@ -642,34 +680,53 @@ check_join_state(enum crmd_fsa_state cur_state, const char *source)
308170
 
308170
     } else if (cur_state == S_INTEGRATION) {
308170
         if (crmd_join_phase_count(crm_join_welcomed) == 0) {
308170
-            crm_debug("join-%d: Integration of %d peers complete: %s",
308170
-                      current_join_id, crmd_join_phase_count(crm_join_integrated), source);
308170
+            int count = crmd_join_phase_count(crm_join_integrated);
308170
+
308170
+            crm_debug("join-%d: Integration of %d peer%s complete "
308170
+                      CRM_XS " state=%s for=%s",
308170
+                      current_join_id, count, pcmk__plural_s(count),
308170
+                      fsa_state2string(cur_state), source);
308170
             register_fsa_input_before(C_FSA_INTERNAL, I_INTEGRATED, NULL);
308170
             return TRUE;
308170
         }
308170
 
308170
     } else if (cur_state == S_FINALIZE_JOIN) {
308170
         if (is_set(fsa_input_register, R_HAVE_CIB) == FALSE) {
308170
-            crm_debug("join-%d: Delaying I_FINALIZED until we have the CIB", current_join_id);
308170
+            crm_debug("join-%d: Delaying finalization until we have CIB "
308170
+                      CRM_XS " state=%s for=%s",
308170
+                      current_join_id, fsa_state2string(cur_state), source);
308170
             return TRUE;
308170
 
308170
         } else if (crmd_join_phase_count(crm_join_welcomed) != 0) {
308170
-            crm_debug("join-%d: Still waiting on %d welcomed nodes",
308170
-                      current_join_id, crmd_join_phase_count(crm_join_welcomed));
308170
+            int count = crmd_join_phase_count(crm_join_welcomed);
308170
+
308170
+            crm_debug("join-%d: Still waiting on %d welcomed node%s "
308170
+                      CRM_XS " state=%s for=%s",
308170
+                      current_join_id, count, pcmk__plural_s(count),
308170
+                      fsa_state2string(cur_state), source);
308170
             crmd_join_phase_log(LOG_DEBUG);
308170
 
308170
         } else if (crmd_join_phase_count(crm_join_integrated) != 0) {
308170
-            crm_debug("join-%d: Still waiting on %d integrated nodes",
308170
-                      current_join_id, crmd_join_phase_count(crm_join_integrated));
308170
+            int count = crmd_join_phase_count(crm_join_integrated);
308170
+
308170
+            crm_debug("join-%d: Still waiting on %d integrated node%s "
308170
+                      CRM_XS " state=%s for=%s",
308170
+                      current_join_id, count, pcmk__plural_s(count),
308170
+                      fsa_state2string(cur_state), source);
308170
             crmd_join_phase_log(LOG_DEBUG);
308170
 
308170
         } else if (crmd_join_phase_count(crm_join_finalized) != 0) {
308170
-            crm_debug("join-%d: Still waiting on %d finalized nodes",
308170
-                      current_join_id, crmd_join_phase_count(crm_join_finalized));
308170
+            int count = crmd_join_phase_count(crm_join_finalized);
308170
+
308170
+            crm_debug("join-%d: Still waiting on %d finalized node%s "
308170
+                      CRM_XS " state=%s for=%s",
308170
+                      current_join_id, count, pcmk__plural_s(count),
308170
+                      fsa_state2string(cur_state), source);
308170
             crmd_join_phase_log(LOG_DEBUG);
308170
 
308170
         } else {
308170
-            crm_debug("join-%d complete: %s", current_join_id, source);
308170
+            crm_debug("join-%d: Complete " CRM_XS " state=%s for=%s",
308170
+                      current_join_id, fsa_state2string(cur_state), source);
308170
             register_fsa_input_later(C_FSA_INTERNAL, I_FINALIZED, NULL);
308170
             return TRUE;
308170
         }
308170
-- 
308170
1.8.3.1
308170
308170
308170
From 034b27734d05e8aeddb586f2daaede8314f9516f Mon Sep 17 00:00:00 2001
308170
From: Ken Gaillot <kgaillot@redhat.com>
308170
Date: Fri, 13 Dec 2019 10:39:34 -0600
308170
Subject: [PATCH 6/8] Log: controller: improve CIB status deletion messages
308170
308170
---
308170
 daemons/controld/controld_utils.c | 25 +++++++++++++++++--------
308170
 1 file changed, 17 insertions(+), 8 deletions(-)
308170
308170
diff --git a/daemons/controld/controld_utils.c b/daemons/controld/controld_utils.c
308170
index 3acd488..bb8ace9 100644
308170
--- a/daemons/controld/controld_utils.c
308170
+++ b/daemons/controld/controld_utils.c
308170
@@ -751,14 +751,18 @@ update_dc(xmlNode * msg)
308170
     return TRUE;
308170
 }
308170
 
308170
-#define STATUS_PATH_MAX 512
308170
 static void
308170
 erase_xpath_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
308170
 {
308170
     char *xpath = user_data;
308170
 
308170
-    do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
308170
-                        "Deletion of \"%s\": %s (rc=%d)", xpath, pcmk_strerror(rc), rc);
308170
+    if (rc == 0) {
308170
+        crm_debug("Deletion of '%s' from CIB (via CIB call %d) succeeded",
308170
+                  xpath, call_id);
308170
+    } else {
308170
+        crm_warn("Deletion of '%s' from CIB (via CIB call %d) failed: %s "
308170
+                 CRM_XS " rc=%d", xpath, call_id, pcmk_strerror(rc), rc);
308170
+    }
308170
 }
308170
 
308170
 #define XPATH_STATUS_TAG "//node_state[@uname='%s']/%s"
308170
@@ -766,14 +770,19 @@ erase_xpath_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void
308170
 void
308170
 erase_status_tag(const char *uname, const char *tag, int options)
308170
 {
308170
-    if (fsa_cib_conn && uname) {
308170
+    CRM_CHECK(uname != NULL, return);
308170
+
308170
+    if (fsa_cib_conn == NULL) {
308170
+        crm_warn("Unable to delete CIB '%s' section for node %s: "
308170
+                 "no CIB connection", tag, uname);
308170
+    } else {
308170
         int call_id;
308170
         char *xpath = crm_strdup_printf(XPATH_STATUS_TAG, uname, tag);
308170
 
308170
-        crm_info("Deleting %s status entries for %s " CRM_XS " xpath=%s",
308170
-                 tag, uname, xpath);
308170
-        call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, xpath, NULL,
308170
-                                             cib_quorum_override | cib_xpath | options);
308170
+        options |= cib_quorum_override|cib_xpath;
308170
+        call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, xpath, NULL, options);
308170
+        crm_info("Deleting CIB '%s' section for node %s (via CIB call %d) "
308170
+                 CRM_XS " xpath=%s", tag, uname, call_id, xpath);
308170
         fsa_register_cib_callback(call_id, FALSE, xpath, erase_xpath_callback);
308170
         // CIB library handles freeing xpath
308170
     }
308170
-- 
308170
1.8.3.1
308170
308170
308170
From 73510818bc9905dcc130893198590b10c0067425 Mon Sep 17 00:00:00 2001
308170
From: Ken Gaillot <kgaillot@redhat.com>
308170
Date: Fri, 13 Dec 2019 10:36:56 -0600
308170
Subject: [PATCH 7/8] Refactor: controller: move erase_status_tag() to
308170
 controld_based.c
308170
308170
---
308170
 daemons/controld/controld_based.c | 38 ++++++++++++++++++++++++++++++++++++++
308170
 daemons/controld/controld_utils.c | 37 -------------------------------------
308170
 2 files changed, 38 insertions(+), 37 deletions(-)
308170
308170
diff --git a/daemons/controld/controld_based.c b/daemons/controld/controld_based.c
308170
index e6a4612..1db5650 100644
308170
--- a/daemons/controld/controld_based.c
308170
+++ b/daemons/controld/controld_based.c
308170
@@ -168,3 +168,41 @@ controld_action_is_recordable(const char *action)
308170
     }
308170
     return TRUE;
308170
 }
308170
+
308170
+static void
308170
+erase_xpath_callback(xmlNode *msg, int call_id, int rc, xmlNode *output,
308170
+                     void *user_data)
308170
+{
308170
+    char *xpath = user_data;
308170
+
308170
+    if (rc == 0) {
308170
+        crm_debug("Deletion of '%s' from CIB (via CIB call %d) succeeded",
308170
+                  xpath, call_id);
308170
+    } else {
308170
+        crm_warn("Deletion of '%s' from CIB (via CIB call %d) failed: %s "
308170
+                 CRM_XS " rc=%d", xpath, call_id, pcmk_strerror(rc), rc);
308170
+    }
308170
+}
308170
+
308170
+#define XPATH_STATUS_TAG "//node_state[@uname='%s']/%s"
308170
+
308170
+void
308170
+erase_status_tag(const char *uname, const char *tag, int options)
308170
+{
308170
+    CRM_CHECK(uname != NULL, return);
308170
+
308170
+    if (fsa_cib_conn == NULL) {
308170
+        crm_warn("Unable to delete CIB '%s' section for node %s: "
308170
+                 "no CIB connection", tag, uname);
308170
+    } else {
308170
+        int call_id;
308170
+        char *xpath = crm_strdup_printf(XPATH_STATUS_TAG, uname, tag);
308170
+
308170
+        options |= cib_quorum_override|cib_xpath;
308170
+        call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, xpath, NULL, options);
308170
+        crm_info("Deleting CIB '%s' section for node %s (via CIB call %d) "
308170
+                 CRM_XS " xpath=%s", tag, uname, call_id, xpath);
308170
+        fsa_register_cib_callback(call_id, FALSE, xpath, erase_xpath_callback);
308170
+        // CIB library handles freeing xpath
308170
+    }
308170
+}
308170
diff --git a/daemons/controld/controld_utils.c b/daemons/controld/controld_utils.c
308170
index bb8ace9..4ed6aeb 100644
308170
--- a/daemons/controld/controld_utils.c
308170
+++ b/daemons/controld/controld_utils.c
308170
@@ -751,43 +751,6 @@ update_dc(xmlNode * msg)
308170
     return TRUE;
308170
 }
308170
 
308170
-static void
308170
-erase_xpath_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
308170
-{
308170
-    char *xpath = user_data;
308170
-
308170
-    if (rc == 0) {
308170
-        crm_debug("Deletion of '%s' from CIB (via CIB call %d) succeeded",
308170
-                  xpath, call_id);
308170
-    } else {
308170
-        crm_warn("Deletion of '%s' from CIB (via CIB call %d) failed: %s "
308170
-                 CRM_XS " rc=%d", xpath, call_id, pcmk_strerror(rc), rc);
308170
-    }
308170
-}
308170
-
308170
-#define XPATH_STATUS_TAG "//node_state[@uname='%s']/%s"
308170
-
308170
-void
308170
-erase_status_tag(const char *uname, const char *tag, int options)
308170
-{
308170
-    CRM_CHECK(uname != NULL, return);
308170
-
308170
-    if (fsa_cib_conn == NULL) {
308170
-        crm_warn("Unable to delete CIB '%s' section for node %s: "
308170
-                 "no CIB connection", tag, uname);
308170
-    } else {
308170
-        int call_id;
308170
-        char *xpath = crm_strdup_printf(XPATH_STATUS_TAG, uname, tag);
308170
-
308170
-        options |= cib_quorum_override|cib_xpath;
308170
-        call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, xpath, NULL, options);
308170
-        crm_info("Deleting CIB '%s' section for node %s (via CIB call %d) "
308170
-                 CRM_XS " xpath=%s", tag, uname, call_id, xpath);
308170
-        fsa_register_cib_callback(call_id, FALSE, xpath, erase_xpath_callback);
308170
-        // CIB library handles freeing xpath
308170
-    }
308170
-}
308170
-
308170
 void crmd_peer_down(crm_node_t *peer, bool full) 
308170
 {
308170
     if(full && peer->state == NULL) {
308170
-- 
308170
1.8.3.1
308170
308170
308170
From c4cc759e733db894957d039f65572cc21704224f Mon Sep 17 00:00:00 2001
308170
From: Ken Gaillot <kgaillot@redhat.com>
308170
Date: Fri, 13 Dec 2019 11:16:25 -0600
308170
Subject: [PATCH 8/8] Refactor: controller: improve efficiency when deleting
308170
 node state
308170
308170
Rename erase_status_xpath() to controld_delete_node_state() to follow current
308170
naming practice.
308170
308170
Instead of passing it a node_state subsection name, pass a new enum value
308170
indicating what to erase (resource history, transient node attributes, or
308170
both). This allows us to improve the log messages further, as well as improving
308170
efficiency when both need to be cleared.
308170
---
308170
 daemons/controld/controld_based.c     | 69 +++++++++++++++++++++++++++--------
308170
 daemons/controld/controld_callbacks.c |  8 +++-
308170
 daemons/controld/controld_execd.c     |  3 +-
308170
 daemons/controld/controld_fencing.c   |  5 +--
308170
 daemons/controld/controld_join_dc.c   |  3 +-
308170
 daemons/controld/controld_remote_ra.c | 24 ++++++------
308170
 daemons/controld/controld_utils.h     | 11 +++++-
308170
 7 files changed, 87 insertions(+), 36 deletions(-)
308170
308170
diff --git a/daemons/controld/controld_based.c b/daemons/controld/controld_based.c
308170
index 1db5650..008a02d 100644
308170
--- a/daemons/controld/controld_based.c
308170
+++ b/daemons/controld/controld_based.c
308170
@@ -170,39 +170,76 @@ controld_action_is_recordable(const char *action)
308170
 }
308170
 
308170
 static void
308170
-erase_xpath_callback(xmlNode *msg, int call_id, int rc, xmlNode *output,
308170
-                     void *user_data)
308170
+cib_delete_callback(xmlNode *msg, int call_id, int rc, xmlNode *output,
308170
+                    void *user_data)
308170
 {
308170
-    char *xpath = user_data;
308170
+    char *desc = user_data;
308170
 
308170
     if (rc == 0) {
308170
-        crm_debug("Deletion of '%s' from CIB (via CIB call %d) succeeded",
308170
-                  xpath, call_id);
308170
+        crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id);
308170
     } else {
308170
-        crm_warn("Deletion of '%s' from CIB (via CIB call %d) failed: %s "
308170
-                 CRM_XS " rc=%d", xpath, call_id, pcmk_strerror(rc), rc);
308170
+        crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d",
308170
+                 desc, call_id, pcmk_strerror(rc), rc);
308170
     }
308170
 }
308170
 
308170
-#define XPATH_STATUS_TAG "//node_state[@uname='%s']/%s"
308170
+// Searches for various portions of node_state to delete
308170
 
308170
+// Match a particular node's node_state (takes node name 1x)
308170
+#define XPATH_NODE_STATE        "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"
308170
+
308170
+// Node's lrm section (name 1x)
308170
+#define XPATH_NODE_LRM          XPATH_NODE_STATE "/" XML_CIB_TAG_LRM
308170
+
308170
+// Node's transient_attributes section (name 1x)
308170
+#define XPATH_NODE_ATTRS        XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS
308170
+
308170
+// Everything under node_state (name 1x)
308170
+#define XPATH_NODE_ALL          XPATH_NODE_STATE "/*"
308170
+
308170
+/*!
308170
+ * \internal
308170
+ * \brief Delete subsection of a node's CIB node_state
308170
+ *
308170
+ * \param[in] uname    Desired node
308170
+ * \param[in] section  Subsection of node_state to delete
308170
+ * \param[in] options  CIB call options to use
308170
+ */
308170
 void
308170
-erase_status_tag(const char *uname, const char *tag, int options)
308170
+controld_delete_node_state(const char *uname, enum controld_section_e section,
308170
+                           int options)
308170
 {
308170
+    char *xpath = NULL;
308170
+    char *desc = NULL;
308170
+
308170
     CRM_CHECK(uname != NULL, return);
308170
+    switch (section) {
308170
+        case controld_section_lrm:
308170
+            xpath = crm_strdup_printf(XPATH_NODE_LRM, uname);
308170
+            desc = crm_strdup_printf("resource history for node %s", uname);
308170
+            break;
308170
+        case controld_section_attrs:
308170
+            xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname);
308170
+            desc = crm_strdup_printf("transient attributes for node %s", uname);
308170
+            break;
308170
+        case controld_section_all:
308170
+            xpath = crm_strdup_printf(XPATH_NODE_ALL, uname);
308170
+            desc = crm_strdup_printf("all state for node %s", uname);
308170
+            break;
308170
+    }
308170
 
308170
     if (fsa_cib_conn == NULL) {
308170
-        crm_warn("Unable to delete CIB '%s' section for node %s: "
308170
-                 "no CIB connection", tag, uname);
308170
+        crm_warn("Unable to delete %s: no CIB connection", desc);
308170
+        free(desc);
308170
     } else {
308170
         int call_id;
308170
-        char *xpath = crm_strdup_printf(XPATH_STATUS_TAG, uname, tag);
308170
 
308170
         options |= cib_quorum_override|cib_xpath;
308170
         call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, xpath, NULL, options);
308170
-        crm_info("Deleting CIB '%s' section for node %s (via CIB call %d) "
308170
-                 CRM_XS " xpath=%s", tag, uname, call_id, xpath);
308170
-        fsa_register_cib_callback(call_id, FALSE, xpath, erase_xpath_callback);
308170
-        // CIB library handles freeing xpath
308170
+        crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s",
308170
+                 desc, call_id, xpath);
308170
+        fsa_register_cib_callback(call_id, FALSE, desc, cib_delete_callback);
308170
+        // CIB library handles freeing desc
308170
     }
308170
+    free(xpath);
308170
 }
308170
diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c
308170
index 5cbd392..f7e3db2 100644
308170
--- a/daemons/controld/controld_callbacks.c
308170
+++ b/daemons/controld/controld_callbacks.c
308170
@@ -200,14 +200,18 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d
308170
                  * transient attributes intact until it rejoins.
308170
                  */
308170
                 if (compare_version(fsa_our_dc_version, "3.0.9") > 0) {
308170
-                    erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
308170
+                    controld_delete_node_state(node->uname,
308170
+                                               controld_section_attrs,
308170
+                                               cib_scope_local);
308170
                 }
308170
 
308170
             } else if(AM_I_DC) {
308170
                 if (appeared) {
308170
                     te_trigger_stonith_history_sync(FALSE);
308170
                 } else {
308170
-                    erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
308170
+                    controld_delete_node_state(node->uname,
308170
+                                               controld_section_attrs,
308170
+                                               cib_scope_local);
308170
                 }
308170
             }
308170
             break;
308170
diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c
308170
index 46c1958..b7deeae 100644
308170
--- a/daemons/controld/controld_execd.c
308170
+++ b/daemons/controld/controld_execd.c
308170
@@ -1411,7 +1411,8 @@ force_reprobe(lrm_state_t *lrm_state, const char *from_sys,
308170
     }
308170
 
308170
     /* Now delete the copy in the CIB */
308170
-    erase_status_tag(lrm_state->node_name, XML_CIB_TAG_LRM, cib_scope_local);
308170
+    controld_delete_node_state(lrm_state->node_name, controld_section_lrm,
308170
+                               cib_scope_local);
308170
 
308170
     /* Finally, _delete_ the value in pacemaker-attrd -- setting it to FALSE
308170
      * would result in the scheduler sending us back here again
308170
diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c
308170
index d9b1e1e..9897cf3 100644
308170
--- a/daemons/controld/controld_fencing.c
308170
+++ b/daemons/controld/controld_fencing.c
308170
@@ -229,9 +229,8 @@ send_stonith_update(crm_action_t *action, const char *target, const char *uuid)
308170
     /* Make sure it sticks */
308170
     /* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override|cib_scope_local);    */
308170
 
308170
-    erase_status_tag(peer->uname, XML_CIB_TAG_LRM, cib_scope_local);
308170
-    erase_status_tag(peer->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
308170
-
308170
+    controld_delete_node_state(peer->uname, controld_section_all,
308170
+                               cib_scope_local);
308170
     free_xml(node_state);
308170
     return;
308170
 }
308170
diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c
308170
index 54324b2..ac6b430 100644
308170
--- a/daemons/controld/controld_join_dc.c
308170
+++ b/daemons/controld/controld_join_dc.c
308170
@@ -587,7 +587,8 @@ do_dc_join_ack(long long action,
308170
     /* Update CIB with node's current executor state. A new transition will be
308170
      * triggered later, when the CIB notifies us of the change.
308170
      */
308170
-    erase_status_tag(join_from, XML_CIB_TAG_LRM, cib_scope_local);
308170
+    controld_delete_node_state(join_from, controld_section_lrm,
308170
+                               cib_scope_local);
308170
     if (safe_str_eq(join_from, fsa_our_uname)) {
308170
         xmlNode *now_dc_lrmd_state = do_lrm_query(TRUE, fsa_our_uname);
308170
 
308170
diff --git a/daemons/controld/controld_remote_ra.c b/daemons/controld/controld_remote_ra.c
308170
index 4fbae45..2d3dfa7 100644
308170
--- a/daemons/controld/controld_remote_ra.c
308170
+++ b/daemons/controld/controld_remote_ra.c
308170
@@ -181,13 +181,13 @@ remote_node_up(const char *node_name)
308170
     CRM_CHECK(node_name != NULL, return);
308170
     crm_info("Announcing pacemaker_remote node %s", node_name);
308170
 
308170
-    /* Clear node's operation history. The node's transient attributes should
308170
-     * and normally will be cleared when the node leaves, but since remote node
308170
-     * state has a number of corner cases, clear them here as well, to be sure.
308170
+    /* Clear node's entire state (resource history and transient attributes).
308170
+     * The transient attributes should and normally will be cleared when the
308170
+     * node leaves, but since remote node state has a number of corner cases,
308170
+     * clear them here as well, to be sure.
308170
      */
308170
     call_opt = crmd_cib_smart_opt();
308170
-    erase_status_tag(node_name, XML_CIB_TAG_LRM, call_opt);
308170
-    erase_status_tag(node_name, XML_TAG_TRANSIENT_NODEATTRS, call_opt);
308170
+    controld_delete_node_state(node_name, controld_section_all, call_opt);
308170
 
308170
     /* Clear node's probed attribute */
308170
     update_attrd(node_name, CRM_OP_PROBED, NULL, NULL, TRUE);
308170
@@ -252,15 +252,15 @@ remote_node_down(const char *node_name, const enum down_opts opts)
308170
     /* Purge node from attrd's memory */
308170
     update_attrd_remote_node_removed(node_name, NULL);
308170
 
308170
-    /* Purge node's transient attributes */
308170
-    erase_status_tag(node_name, XML_TAG_TRANSIENT_NODEATTRS, call_opt);
308170
-
308170
-    /* Normally, the LRM operation history should be kept until the node comes
308170
-     * back up. However, after a successful fence, we want to clear it, so we
308170
-     * don't think resources are still running on the node.
308170
+    /* Normally, only node attributes should be erased, and the resource history
308170
+     * should be kept until the node comes back up. However, after a successful
308170
+     * fence, we want to clear the history as well, so we don't think resources
308170
+     * are still running on the node.
308170
      */
308170
     if (opts == DOWN_ERASE_LRM) {
308170
-        erase_status_tag(node_name, XML_CIB_TAG_LRM, call_opt);
308170
+        controld_delete_node_state(node_name, controld_section_all, call_opt);
308170
+    } else {
308170
+        controld_delete_node_state(node_name, controld_section_attrs, call_opt);
308170
     }
308170
 
308170
     /* Ensure node is in the remote peer cache with lost state */
308170
diff --git a/daemons/controld/controld_utils.h b/daemons/controld/controld_utils.h
308170
index cf04f13..f902361 100644
308170
--- a/daemons/controld/controld_utils.h
308170
+++ b/daemons/controld/controld_utils.h
308170
@@ -70,7 +70,6 @@ xmlNode *create_node_state_update(crm_node_t *node, int flags,
308170
                                   xmlNode *parent, const char *source);
308170
 void populate_cib_nodes(enum node_update_flags flags, const char *source);
308170
 void crm_update_quorum(gboolean quorum, gboolean force_update);
308170
-void erase_status_tag(const char *uname, const char *tag, int options);
308170
 void controld_close_attrd_ipc(void);
308170
 void update_attrd(const char *host, const char *name, const char *value, const char *user_name, gboolean is_remote_node);
308170
 void update_attrd_remote_node_removed(const char *host, const char *user_name);
308170
@@ -87,6 +86,16 @@ unsigned int cib_op_timeout(void);
308170
 bool feature_set_compatible(const char *dc_version, const char *join_version);
308170
 bool controld_action_is_recordable(const char *action);
308170
 
308170
+// Subsections of node_state
308170
+enum controld_section_e {
308170
+    controld_section_lrm,
308170
+    controld_section_attrs,
308170
+    controld_section_all,
308170
+};
308170
+
308170
+void controld_delete_node_state(const char *uname,
308170
+                                enum controld_section_e section, int options);
308170
+
308170
 const char *get_node_id(xmlNode *lrm_rsc_op);
308170
 
308170
 /* Convenience macro for registering a CIB callback
308170
-- 
308170
1.8.3.1
308170