fd9fba
From de05f6b52c667155d262ceeb541dc1041d079d71 Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Thu, 8 Sep 2022 11:36:58 -0400
fd9fba
Subject: [PATCH 01/26] Refactor: tools: Use a uint32_t for attr_options.
fd9fba
fd9fba
---
fd9fba
 tools/attrd_updater.c | 2 +-
fd9fba
 1 file changed, 1 insertion(+), 1 deletion(-)
fd9fba
fd9fba
diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c
fd9fba
index d90567a..b85a281 100644
fd9fba
--- a/tools/attrd_updater.c
fd9fba
+++ b/tools/attrd_updater.c
fd9fba
@@ -47,7 +47,7 @@ struct {
fd9fba
     gchar *attr_node;
fd9fba
     gchar *attr_set;
fd9fba
     char *attr_value;
fd9fba
-    int attr_options;
fd9fba
+    uint32_t attr_options;
fd9fba
     gboolean query_all;
fd9fba
     gboolean quiet;
fd9fba
 } options = {
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From c6637520b474d44553ade52c0dbe9e36e873135f Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Fri, 21 Oct 2022 14:31:16 -0400
fd9fba
Subject: [PATCH 02/26] Refactor: libcrmcommon: Make pcmk__xe_match more
fd9fba
 broadly useful.
fd9fba
fd9fba
If attr_v is NULL, simply return the first node with a matching name.
fd9fba
---
fd9fba
 lib/common/xml.c | 10 ++++++----
fd9fba
 1 file changed, 6 insertions(+), 4 deletions(-)
fd9fba
fd9fba
diff --git a/lib/common/xml.c b/lib/common/xml.c
fd9fba
index 036dd87..ac6f46a 100644
fd9fba
--- a/lib/common/xml.c
fd9fba
+++ b/lib/common/xml.c
fd9fba
@@ -510,7 +510,7 @@ find_xml_node(const xmlNode *root, const char *search_path, gboolean must_find)
fd9fba
  * \param[in] parent     XML element to search
fd9fba
  * \param[in] node_name  If not NULL, only match children of this type
fd9fba
  * \param[in] attr_n     If not NULL, only match children with an attribute
fd9fba
- *                       of this name and a value of \p attr_v
fd9fba
+ *                       of this name.
fd9fba
  * \param[in] attr_v     If \p attr_n and this are not NULL, only match children
fd9fba
  *                       with an attribute named \p attr_n and this value
fd9fba
  *
fd9fba
@@ -520,14 +520,16 @@ xmlNode *
fd9fba
 pcmk__xe_match(const xmlNode *parent, const char *node_name,
fd9fba
                const char *attr_n, const char *attr_v)
fd9fba
 {
fd9fba
-    /* ensure attr_v specified when attr_n is */
fd9fba
-    CRM_CHECK(attr_n == NULL || attr_v != NULL, return NULL);
fd9fba
+    CRM_CHECK(parent != NULL, return NULL);
fd9fba
+    CRM_CHECK(attr_v == NULL || attr_n != NULL, return NULL);
fd9fba
 
fd9fba
     for (xmlNode *child = pcmk__xml_first_child(parent); child != NULL;
fd9fba
          child = pcmk__xml_next(child)) {
fd9fba
         if (pcmk__str_eq(node_name, (const char *) (child->name),
fd9fba
                          pcmk__str_null_matches)
fd9fba
-            && ((attr_n == NULL) || attr_matches(child, attr_n, attr_v))) {
fd9fba
+            && ((attr_n == NULL) ||
fd9fba
+                (attr_v == NULL && xmlHasProp(child, (pcmkXmlStr) attr_n)) ||
fd9fba
+                (attr_v != NULL && attr_matches(child, attr_n, attr_v)))) {
fd9fba
             return child;
fd9fba
         }
fd9fba
     }
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From dd520579484c6ec091f7fbb550347941302dad0e Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Fri, 21 Oct 2022 14:32:46 -0400
fd9fba
Subject: [PATCH 03/26] Tests: libcrmcommon: Add tests for pcmk__xe_match.
fd9fba
fd9fba
---
fd9fba
 lib/common/tests/xml/Makefile.am           |   3 +-
fd9fba
 lib/common/tests/xml/pcmk__xe_match_test.c | 105 +++++++++++++++++++++
fd9fba
 2 files changed, 107 insertions(+), 1 deletion(-)
fd9fba
 create mode 100644 lib/common/tests/xml/pcmk__xe_match_test.c
fd9fba
fd9fba
diff --git a/lib/common/tests/xml/Makefile.am b/lib/common/tests/xml/Makefile.am
fd9fba
index 342ca07..0ccdcc3 100644
fd9fba
--- a/lib/common/tests/xml/Makefile.am
fd9fba
+++ b/lib/common/tests/xml/Makefile.am
fd9fba
@@ -11,6 +11,7 @@ include $(top_srcdir)/mk/tap.mk
fd9fba
 include $(top_srcdir)/mk/unittest.mk
fd9fba
 
fd9fba
 # Add "_test" to the end of all test program names to simplify .gitignore.
fd9fba
-check_PROGRAMS =	pcmk__xe_foreach_child_test
fd9fba
+check_PROGRAMS =	pcmk__xe_foreach_child_test \
fd9fba
+					pcmk__xe_match_test
fd9fba
 
fd9fba
 TESTS = $(check_PROGRAMS)
fd9fba
diff --git a/lib/common/tests/xml/pcmk__xe_match_test.c b/lib/common/tests/xml/pcmk__xe_match_test.c
fd9fba
new file mode 100644
fd9fba
index 0000000..fd529ba
fd9fba
--- /dev/null
fd9fba
+++ b/lib/common/tests/xml/pcmk__xe_match_test.c
fd9fba
@@ -0,0 +1,105 @@
fd9fba
+/*
fd9fba
+ * Copyright 2022 the Pacemaker project contributors
fd9fba
+ *
fd9fba
+ * The version control history for this file may have further details.
fd9fba
+ *
fd9fba
+ * This source code is licensed under the GNU Lesser General Public License
fd9fba
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
fd9fba
+ */
fd9fba
+
fd9fba
+#include <crm_internal.h>
fd9fba
+
fd9fba
+#include <crm/common/unittest_internal.h>
fd9fba
+#include <crm/common/xml_internal.h>
fd9fba
+
fd9fba
+const char *str1 =
fd9fba
+    "<xml>\n"
fd9fba
+    "  \n"
fd9fba
+    "  <nodeA attrA=\"123\" id=\"1\">\n"
fd9fba
+    "    content\n"
fd9fba
+    "  </nodeA>\n"
fd9fba
+    "  \n"
fd9fba
+    "  <nodeA attrA=\"456\" id=\"2\">\n"
fd9fba
+    "    content\n"
fd9fba
+    "  </nodeA>\n"
fd9fba
+    "  \n"
fd9fba
+    "  <nodeA attrB=\"XYZ\" id=\"3\">\n"
fd9fba
+    "    content\n"
fd9fba
+    "  </nodeA>\n"
fd9fba
+    "  \n"
fd9fba
+    "  <nodeB attrA=\"123\" id=\"4\">\n"
fd9fba
+    "    content\n"
fd9fba
+    "  </nodeA>\n"
fd9fba
+    "  \n"
fd9fba
+    "  <nodeB attrB=\"ABC\" id=\"5\">\n"
fd9fba
+    "    content\n"
fd9fba
+    "  </nodeA>\n"
fd9fba
+    "</xml>";
fd9fba
+
fd9fba
+static void
fd9fba
+bad_input(void **state) {
fd9fba
+    xmlNode *xml = string2xml(str1);
fd9fba
+
fd9fba
+    assert_null(pcmk__xe_match(NULL, NULL, NULL, NULL));
fd9fba
+    assert_null(pcmk__xe_match(NULL, NULL, NULL, "attrX"));
fd9fba
+
fd9fba
+    free_xml(xml);
fd9fba
+}
fd9fba
+
fd9fba
+static void
fd9fba
+not_found(void **state) {
fd9fba
+    xmlNode *xml = string2xml(str1);
fd9fba
+
fd9fba
+    /* No node with an attrX attribute */
fd9fba
+    assert_null(pcmk__xe_match(xml, NULL, "attrX", NULL));
fd9fba
+    /* No nodeX node */
fd9fba
+    assert_null(pcmk__xe_match(xml, "nodeX", NULL, NULL));
fd9fba
+    /* No nodeA node with attrX */
fd9fba
+    assert_null(pcmk__xe_match(xml, "nodeA", "attrX", NULL));
fd9fba
+    /* No nodeA node with attrA=XYZ */
fd9fba
+    assert_null(pcmk__xe_match(xml, "nodeA", "attrA", "XYZ"));
fd9fba
+
fd9fba
+    free_xml(xml);
fd9fba
+}
fd9fba
+
fd9fba
+static void
fd9fba
+find_attrB(void **state) {
fd9fba
+    xmlNode *xml = string2xml(str1);
fd9fba
+    xmlNode *result = NULL;
fd9fba
+
fd9fba
+    /* Find the first node with attrB */
fd9fba
+    result = pcmk__xe_match(xml, NULL, "attrB", NULL);
fd9fba
+    assert_non_null(result);
fd9fba
+    assert_string_equal(crm_element_value(result, "id"), "3");
fd9fba
+
fd9fba
+    /* Find the first nodeB with attrB */
fd9fba
+    result = pcmk__xe_match(xml, "nodeB", "attrB", NULL);
fd9fba
+    assert_non_null(result);
fd9fba
+    assert_string_equal(crm_element_value(result, "id"), "5");
fd9fba
+
fd9fba
+    free_xml(xml);
fd9fba
+}
fd9fba
+
fd9fba
+static void
fd9fba
+find_attrA_matching(void **state) {
fd9fba
+    xmlNode *xml = string2xml(str1);
fd9fba
+    xmlNode *result = NULL;
fd9fba
+
fd9fba
+    /* Find attrA=456 */
fd9fba
+    result = pcmk__xe_match(xml, NULL, "attrA", "456");
fd9fba
+    assert_non_null(result);
fd9fba
+    assert_string_equal(crm_element_value(result, "id"), "2");
fd9fba
+
fd9fba
+    /* Find a nodeB with attrA=123 */
fd9fba
+    result = pcmk__xe_match(xml, "nodeB", "attrA", "123");
fd9fba
+    assert_non_null(result);
fd9fba
+    assert_string_equal(crm_element_value(result, "id"), "4");
fd9fba
+
fd9fba
+    free_xml(xml);
fd9fba
+}
fd9fba
+
fd9fba
+PCMK__UNIT_TEST(NULL, NULL,
fd9fba
+                cmocka_unit_test(bad_input),
fd9fba
+                cmocka_unit_test(not_found),
fd9fba
+                cmocka_unit_test(find_attrB),
fd9fba
+                cmocka_unit_test(find_attrA_matching));
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 03af8498d8aaf21c509cec9b0ec4b78475da41d7 Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Thu, 8 Sep 2022 12:22:26 -0400
fd9fba
Subject: [PATCH 04/26] Feature: libcrmcommon: Add attrd options for specifying
fd9fba
 a sync point.
fd9fba
fd9fba
---
fd9fba
 include/crm/common/attrd_internal.h | 16 +++++++++-------
fd9fba
 1 file changed, 9 insertions(+), 7 deletions(-)
fd9fba
fd9fba
diff --git a/include/crm/common/attrd_internal.h b/include/crm/common/attrd_internal.h
fd9fba
index f7033ad..389be48 100644
fd9fba
--- a/include/crm/common/attrd_internal.h
fd9fba
+++ b/include/crm/common/attrd_internal.h
fd9fba
@@ -16,13 +16,15 @@ extern "C" {
fd9fba
 
fd9fba
 // Options for clients to use with functions below
fd9fba
 enum pcmk__node_attr_opts {
fd9fba
-    pcmk__node_attr_none    = 0,
fd9fba
-    pcmk__node_attr_remote  = (1 << 0),
fd9fba
-    pcmk__node_attr_private = (1 << 1),
fd9fba
-    pcmk__node_attr_pattern = (1 << 2),
fd9fba
-    pcmk__node_attr_value   = (1 << 3),
fd9fba
-    pcmk__node_attr_delay   = (1 << 4),
fd9fba
-    pcmk__node_attr_perm    = (1 << 5),
fd9fba
+    pcmk__node_attr_none           = 0,
fd9fba
+    pcmk__node_attr_remote         = (1 << 0),
fd9fba
+    pcmk__node_attr_private        = (1 << 1),
fd9fba
+    pcmk__node_attr_pattern        = (1 << 2),
fd9fba
+    pcmk__node_attr_value          = (1 << 3),
fd9fba
+    pcmk__node_attr_delay          = (1 << 4),
fd9fba
+    pcmk__node_attr_perm           = (1 << 5),
fd9fba
+    pcmk__node_attr_sync_local     = (1 << 6),
fd9fba
+    pcmk__node_attr_sync_cluster   = (1 << 7),
fd9fba
 };
fd9fba
 
fd9fba
 #define pcmk__set_node_attr_flags(node_attr_flags, flags_to_set) do {   \
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 5c8825293ee21d3823bdcd01b0df9c7d39739940 Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Thu, 8 Sep 2022 12:23:09 -0400
fd9fba
Subject: [PATCH 05/26] Feature: libcrmcommon: Add sync point to IPC request
fd9fba
 XML.
fd9fba
fd9fba
If one of the pcmk__node_attr_sync_* options is provided, add an
fd9fba
attribute to the request XML.  This will later be inspected by the
fd9fba
server to determine when to send the reply to the client.
fd9fba
---
fd9fba
 include/crm/common/options_internal.h | 2 ++
fd9fba
 include/crm_internal.h                | 1 +
fd9fba
 lib/common/ipc_attrd.c                | 6 ++++++
fd9fba
 3 files changed, 9 insertions(+)
fd9fba
fd9fba
diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h
fd9fba
index b153c67..f29ba3f 100644
fd9fba
--- a/include/crm/common/options_internal.h
fd9fba
+++ b/include/crm/common/options_internal.h
fd9fba
@@ -145,9 +145,11 @@ bool pcmk__valid_sbd_timeout(const char *value);
fd9fba
 #define PCMK__META_ALLOW_UNHEALTHY_NODES    "allow-unhealthy-nodes"
fd9fba
 
fd9fba
 // Constants for enumerated values for various options
fd9fba
+#define PCMK__VALUE_CLUSTER                 "cluster"
fd9fba
 #define PCMK__VALUE_CUSTOM                  "custom"
fd9fba
 #define PCMK__VALUE_FENCING                 "fencing"
fd9fba
 #define PCMK__VALUE_GREEN                   "green"
fd9fba
+#define PCMK__VALUE_LOCAL                   "local"
fd9fba
 #define PCMK__VALUE_MIGRATE_ON_RED          "migrate-on-red"
fd9fba
 #define PCMK__VALUE_NONE                    "none"
fd9fba
 #define PCMK__VALUE_NOTHING                 "nothing"
fd9fba
diff --git a/include/crm_internal.h b/include/crm_internal.h
fd9fba
index e6e2e96..08193c3 100644
fd9fba
--- a/include/crm_internal.h
fd9fba
+++ b/include/crm_internal.h
fd9fba
@@ -71,6 +71,7 @@
fd9fba
 #define PCMK__XA_ATTR_RESOURCE          "attr_resource"
fd9fba
 #define PCMK__XA_ATTR_SECTION           "attr_section"
fd9fba
 #define PCMK__XA_ATTR_SET               "attr_set"
fd9fba
+#define PCMK__XA_ATTR_SYNC_POINT        "attr_sync_point"
fd9fba
 #define PCMK__XA_ATTR_USER              "attr_user"
fd9fba
 #define PCMK__XA_ATTR_UUID              "attr_key"
fd9fba
 #define PCMK__XA_ATTR_VALUE             "attr_value"
fd9fba
diff --git a/lib/common/ipc_attrd.c b/lib/common/ipc_attrd.c
fd9fba
index f6cfbc4..4606509 100644
fd9fba
--- a/lib/common/ipc_attrd.c
fd9fba
+++ b/lib/common/ipc_attrd.c
fd9fba
@@ -431,6 +431,12 @@ populate_update_op(xmlNode *op, const char *node, const char *name, const char *
fd9fba
                     pcmk_is_set(options, pcmk__node_attr_remote));
fd9fba
     crm_xml_add_int(op, PCMK__XA_ATTR_IS_PRIVATE,
fd9fba
                     pcmk_is_set(options, pcmk__node_attr_private));
fd9fba
+
fd9fba
+    if (pcmk_is_set(options, pcmk__node_attr_sync_local)) {
fd9fba
+        crm_xml_add(op, PCMK__XA_ATTR_SYNC_POINT, PCMK__VALUE_LOCAL);
fd9fba
+    } else if (pcmk_is_set(options, pcmk__node_attr_sync_cluster)) {
fd9fba
+        crm_xml_add(op, PCMK__XA_ATTR_SYNC_POINT, PCMK__VALUE_CLUSTER);
fd9fba
+    }
fd9fba
 }
fd9fba
 
fd9fba
 int
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From e2b3fee630caf0846ca8bbffcef4d6d2acfd32a5 Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Thu, 8 Sep 2022 12:26:28 -0400
fd9fba
Subject: [PATCH 06/26] Feature: tools: Add --wait= parameter to attrd_updater.
fd9fba
fd9fba
This command line option is used to specify the sync point to use.  For
fd9fba
the moment, it has no effect.
fd9fba
---
fd9fba
 tools/attrd_updater.c | 24 ++++++++++++++++++++++++
fd9fba
 1 file changed, 24 insertions(+)
fd9fba
fd9fba
diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c
fd9fba
index b85a281..c4779a6 100644
fd9fba
--- a/tools/attrd_updater.c
fd9fba
+++ b/tools/attrd_updater.c
fd9fba
@@ -97,6 +97,22 @@ section_cb (const gchar *option_name, const gchar *optarg, gpointer data, GError
fd9fba
     return TRUE;
fd9fba
 }
fd9fba
 
fd9fba
+static gboolean
fd9fba
+wait_cb (const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
fd9fba
+    if (pcmk__str_eq(optarg, "no", pcmk__str_none)) {
fd9fba
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
fd9fba
+        return TRUE;
fd9fba
+    } else if (pcmk__str_eq(optarg, PCMK__VALUE_LOCAL, pcmk__str_none)) {
fd9fba
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
fd9fba
+        pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local);
fd9fba
+        return TRUE;
fd9fba
+    } else {
fd9fba
+        g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_USAGE,
fd9fba
+                    "--wait= must be one of 'no', 'local', 'cluster'");
fd9fba
+        return FALSE;
fd9fba
+    }
fd9fba
+}
fd9fba
+
fd9fba
 #define INDENT "                              "
fd9fba
 
fd9fba
 static GOptionEntry required_entries[] = {
fd9fba
@@ -175,6 +191,14 @@ static GOptionEntry addl_entries[] = {
fd9fba
       "If this creates a new attribute, never write the attribute to CIB",
fd9fba
       NULL },
fd9fba
 
fd9fba
+    { "wait", 'W', 0, G_OPTION_ARG_CALLBACK, wait_cb,
fd9fba
+      "Wait for some event to occur before returning.  Values are 'no' (wait\n"
fd9fba
+      INDENT "only for the attribute daemon to acknowledge the request) or\n"
fd9fba
+      INDENT "'local' (wait until the change has propagated to where a local\n"
fd9fba
+      INDENT "query will return the request value, or the value set by a\n"
fd9fba
+      INDENT "later request).  Default is 'no'.",
fd9fba
+      "UNTIL" },
fd9fba
+
fd9fba
     { NULL }
fd9fba
 };
fd9fba
 
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 52d51ab41b2f00e72724ab39835b3db86605a96b Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Thu, 20 Oct 2022 14:40:13 -0400
fd9fba
Subject: [PATCH 07/26] Feature: daemons: Add functions for checking a request
fd9fba
 for a sync point.
fd9fba
fd9fba
---
fd9fba
 daemons/attrd/Makefile.am       |  1 +
fd9fba
 daemons/attrd/attrd_sync.c      | 38 +++++++++++++++++++++++++++++++++
fd9fba
 daemons/attrd/pacemaker-attrd.h |  3 +++
fd9fba
 3 files changed, 42 insertions(+)
fd9fba
 create mode 100644 daemons/attrd/attrd_sync.c
fd9fba
fd9fba
diff --git a/daemons/attrd/Makefile.am b/daemons/attrd/Makefile.am
fd9fba
index 1a3d360..6bb81c4 100644
fd9fba
--- a/daemons/attrd/Makefile.am
fd9fba
+++ b/daemons/attrd/Makefile.am
fd9fba
@@ -32,6 +32,7 @@ pacemaker_attrd_SOURCES	= attrd_alerts.c 	\
fd9fba
 						  attrd_elections.c \
fd9fba
 						  attrd_ipc.c 		\
fd9fba
 						  attrd_messages.c 		\
fd9fba
+						  attrd_sync.c 		\
fd9fba
 						  attrd_utils.c 	\
fd9fba
 						  pacemaker-attrd.c
fd9fba
 
fd9fba
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fd9fba
new file mode 100644
fd9fba
index 0000000..92759d2
fd9fba
--- /dev/null
fd9fba
+++ b/daemons/attrd/attrd_sync.c
fd9fba
@@ -0,0 +1,38 @@
fd9fba
+/*
fd9fba
+ * Copyright 2022 the Pacemaker project contributors
fd9fba
+ *
fd9fba
+ * The version control history for this file may have further details.
fd9fba
+ *
fd9fba
+ * This source code is licensed under the GNU General Public License version 2
fd9fba
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
fd9fba
+ */
fd9fba
+
fd9fba
+#include <crm_internal.h>
fd9fba
+
fd9fba
+#include <crm/msg_xml.h>
fd9fba
+#include <crm/common/attrd_internal.h>
fd9fba
+
fd9fba
+#include "pacemaker-attrd.h"
fd9fba
+
fd9fba
+const char *
fd9fba
+attrd_request_sync_point(xmlNode *xml)
fd9fba
+{
fd9fba
+    if (xml_has_children(xml)) {
fd9fba
+        xmlNode *child = pcmk__xe_match(xml, XML_ATTR_OP, PCMK__XA_ATTR_SYNC_POINT, NULL);
fd9fba
+
fd9fba
+        if (child) {
fd9fba
+            return crm_element_value(child, PCMK__XA_ATTR_SYNC_POINT);
fd9fba
+        } else {
fd9fba
+            return NULL;
fd9fba
+        }
fd9fba
+
fd9fba
+    } else {
fd9fba
+        return crm_element_value(xml, PCMK__XA_ATTR_SYNC_POINT);
fd9fba
+    }
fd9fba
+}
fd9fba
+
fd9fba
+bool
fd9fba
+attrd_request_has_sync_point(xmlNode *xml)
fd9fba
+{
fd9fba
+    return attrd_request_sync_point(xml) != NULL;
fd9fba
+}
fd9fba
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fd9fba
index 71ce90a..ff850bb 100644
fd9fba
--- a/daemons/attrd/pacemaker-attrd.h
fd9fba
+++ b/daemons/attrd/pacemaker-attrd.h
fd9fba
@@ -182,4 +182,7 @@ mainloop_timer_t *attrd_add_timer(const char *id, int timeout_ms, attribute_t *a
fd9fba
 void attrd_unregister_handlers(void);
fd9fba
 void attrd_handle_request(pcmk__request_t *request);
fd9fba
 
fd9fba
+const char *attrd_request_sync_point(xmlNode *xml);
fd9fba
+bool attrd_request_has_sync_point(xmlNode *xml);
fd9fba
+
fd9fba
 #endif /* PACEMAKER_ATTRD__H */
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 2e0509a12ee7d4a612133ee65b75245eea7d271d Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Thu, 20 Oct 2022 14:42:04 -0400
fd9fba
Subject: [PATCH 08/26] Refactor: daemons: Don't ACK update requests that give
fd9fba
 a sync point.
fd9fba
fd9fba
The ACK is the only response from the server for update messages.  If
fd9fba
the message specified that it wanted to wait for a sync point, we need
fd9fba
to delay sending that response until the sync point is reached.
fd9fba
Therefore, do not always immediately send the ACK.
fd9fba
---
fd9fba
 daemons/attrd/attrd_messages.c | 19 ++++++++++++++-----
fd9fba
 1 file changed, 14 insertions(+), 5 deletions(-)
fd9fba
fd9fba
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fd9fba
index de4a28a..9e8ae40 100644
fd9fba
--- a/daemons/attrd/attrd_messages.c
fd9fba
+++ b/daemons/attrd/attrd_messages.c
fd9fba
@@ -137,12 +137,21 @@ handle_update_request(pcmk__request_t *request)
fd9fba
         attrd_peer_update(peer, request->xml, host, false);
fd9fba
         pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fd9fba
         return NULL;
fd9fba
+
fd9fba
     } else {
fd9fba
-        /* Because attrd_client_update can be called recursively, we send the ACK
fd9fba
-         * here to ensure that the client only ever receives one.
fd9fba
-         */
fd9fba
-        attrd_send_ack(request->ipc_client, request->ipc_id,
fd9fba
-                       request->flags|crm_ipc_client_response);
fd9fba
+        if (!attrd_request_has_sync_point(request->xml)) {
fd9fba
+            /* If the client doesn't want to wait for a sync point, go ahead and send
fd9fba
+             * the ACK immediately.  Otherwise, we'll send the ACK when the appropriate
fd9fba
+             * sync point is reached.
fd9fba
+             *
fd9fba
+             * In the normal case, attrd_client_update can be called recursively which
fd9fba
+             * makes where to send the ACK tricky.  Doing it here ensures the client
fd9fba
+             * only ever receives one.
fd9fba
+             */
fd9fba
+            attrd_send_ack(request->ipc_client, request->ipc_id,
fd9fba
+                           request->flags|crm_ipc_client_response);
fd9fba
+        }
fd9fba
+
fd9fba
         return attrd_client_update(request);
fd9fba
     }
fd9fba
 }
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 2a0ff66cdf0085c4c8ab1992ef7e785a4facc8c7 Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Thu, 20 Oct 2022 14:48:48 -0400
fd9fba
Subject: [PATCH 09/26] Feature: daemons: Add support for local sync points on
fd9fba
 updates.
fd9fba
fd9fba
In the IPC dispatcher for attrd, add the client to a wait list if its
fd9fba
request specifies a sync point.  When the attribute's value is changed
fd9fba
on the local attrd, alert any clients waiting on a local sync point by
fd9fba
then sending the previously delayed ACK.
fd9fba
fd9fba
Sync points for other requests and the global sync point are not yet
fd9fba
supported.
fd9fba
fd9fba
Fixes T35.
fd9fba
---
fd9fba
 daemons/attrd/attrd_corosync.c  |  18 +++++
fd9fba
 daemons/attrd/attrd_messages.c  |  12 ++-
fd9fba
 daemons/attrd/attrd_sync.c      | 137 ++++++++++++++++++++++++++++++++
fd9fba
 daemons/attrd/pacemaker-attrd.h |   7 ++
fd9fba
 4 files changed, 173 insertions(+), 1 deletion(-)
fd9fba
fd9fba
diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
fd9fba
index 539e5bf..4337280 100644
fd9fba
--- a/daemons/attrd/attrd_corosync.c
fd9fba
+++ b/daemons/attrd/attrd_corosync.c
fd9fba
@@ -568,14 +568,32 @@ void
fd9fba
 attrd_peer_update(const crm_node_t *peer, xmlNode *xml, const char *host,
fd9fba
                   bool filter)
fd9fba
 {
fd9fba
+    bool handle_sync_point = false;
fd9fba
+
fd9fba
     if (xml_has_children(xml)) {
fd9fba
         for (xmlNode *child = first_named_child(xml, XML_ATTR_OP); child != NULL;
fd9fba
              child = crm_next_same_xml(child)) {
fd9fba
             copy_attrs(xml, child);
fd9fba
             attrd_peer_update_one(peer, child, filter);
fd9fba
+
fd9fba
+            if (attrd_request_has_sync_point(child)) {
fd9fba
+                handle_sync_point = true;
fd9fba
+            }
fd9fba
         }
fd9fba
 
fd9fba
     } else {
fd9fba
         attrd_peer_update_one(peer, xml, filter);
fd9fba
+
fd9fba
+        if (attrd_request_has_sync_point(xml)) {
fd9fba
+            handle_sync_point = true;
fd9fba
+        }
fd9fba
+    }
fd9fba
+
fd9fba
+    /* If the update XML specified that the client wanted to wait for a sync
fd9fba
+     * point, process that now.
fd9fba
+     */
fd9fba
+    if (handle_sync_point) {
fd9fba
+        crm_debug("Hit local sync point for attribute update");
fd9fba
+        attrd_ack_waitlist_clients(attrd_sync_point_local, xml);
fd9fba
     }
fd9fba
 }
fd9fba
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fd9fba
index 9e8ae40..c96700f 100644
fd9fba
--- a/daemons/attrd/attrd_messages.c
fd9fba
+++ b/daemons/attrd/attrd_messages.c
fd9fba
@@ -139,7 +139,17 @@ handle_update_request(pcmk__request_t *request)
fd9fba
         return NULL;
fd9fba
 
fd9fba
     } else {
fd9fba
-        if (!attrd_request_has_sync_point(request->xml)) {
fd9fba
+        if (attrd_request_has_sync_point(request->xml)) {
fd9fba
+            /* If this client supplied a sync point it wants to wait for, add it to
fd9fba
+             * the wait list.  Clients on this list will not receive an ACK until
fd9fba
+             * their sync point is hit which will result in the client stalled there
fd9fba
+             * until it receives a response.
fd9fba
+             *
fd9fba
+             * All other clients will receive the expected response as normal.
fd9fba
+             */
fd9fba
+            attrd_add_client_to_waitlist(request);
fd9fba
+
fd9fba
+        } else {
fd9fba
             /* If the client doesn't want to wait for a sync point, go ahead and send
fd9fba
              * the ACK immediately.  Otherwise, we'll send the ACK when the appropriate
fd9fba
              * sync point is reached.
fd9fba
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fd9fba
index 92759d2..2981bd0 100644
fd9fba
--- a/daemons/attrd/attrd_sync.c
fd9fba
+++ b/daemons/attrd/attrd_sync.c
fd9fba
@@ -14,6 +14,143 @@
fd9fba
 
fd9fba
 #include "pacemaker-attrd.h"
fd9fba
 
fd9fba
+/* A hash table storing clients that are waiting on a sync point to be reached.
fd9fba
+ * The key is waitlist_client - just a plain int.  The obvious key would be
fd9fba
+ * the IPC client's ID, but this is not guaranteed to be unique.  A single client
fd9fba
+ * could be waiting on a sync point for multiple attributes at the same time.
fd9fba
+ *
fd9fba
+ * It is not expected that this hash table will ever be especially large.
fd9fba
+ */
fd9fba
+static GHashTable *waitlist = NULL;
fd9fba
+static int waitlist_client = 0;
fd9fba
+
fd9fba
+struct waitlist_node {
fd9fba
+    /* What kind of sync point does this node describe? */
fd9fba
+    enum attrd_sync_point sync_point;
fd9fba
+
fd9fba
+    /* Information required to construct and send a reply to the client. */
fd9fba
+    char *client_id;
fd9fba
+    uint32_t ipc_id;
fd9fba
+    uint32_t flags;
fd9fba
+};
fd9fba
+
fd9fba
+static void
fd9fba
+next_key(void)
fd9fba
+{
fd9fba
+    do {
fd9fba
+        waitlist_client++;
fd9fba
+        if (waitlist_client < 0) {
fd9fba
+            waitlist_client = 1;
fd9fba
+        }
fd9fba
+    } while (g_hash_table_contains(waitlist, GINT_TO_POINTER(waitlist_client)));
fd9fba
+}
fd9fba
+
fd9fba
+static void
fd9fba
+free_waitlist_node(gpointer data)
fd9fba
+{
fd9fba
+    struct waitlist_node *wl = (struct waitlist_node *) data;
fd9fba
+
fd9fba
+    free(wl->client_id);
fd9fba
+    free(wl);
fd9fba
+}
fd9fba
+
fd9fba
+static const char *
fd9fba
+sync_point_str(enum attrd_sync_point sync_point)
fd9fba
+{
fd9fba
+    if (sync_point == attrd_sync_point_local) {
fd9fba
+        return PCMK__VALUE_LOCAL;
fd9fba
+    } else if  (sync_point == attrd_sync_point_cluster) {
fd9fba
+        return PCMK__VALUE_CLUSTER;
fd9fba
+    } else {
fd9fba
+        return "unknown";
fd9fba
+    }
fd9fba
+}
fd9fba
+
fd9fba
+void
fd9fba
+attrd_add_client_to_waitlist(pcmk__request_t *request)
fd9fba
+{
fd9fba
+    const char *sync_point = attrd_request_sync_point(request->xml);
fd9fba
+    struct waitlist_node *wl = NULL;
fd9fba
+
fd9fba
+    if (sync_point == NULL) {
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    if (waitlist == NULL) {
fd9fba
+        waitlist = pcmk__intkey_table(free_waitlist_node);
fd9fba
+    }
fd9fba
+
fd9fba
+    wl = calloc(sizeof(struct waitlist_node), 1);
fd9fba
+
fd9fba
+    CRM_ASSERT(wl != NULL);
fd9fba
+
fd9fba
+    wl->client_id = strdup(request->ipc_client->id);
fd9fba
+
fd9fba
+    CRM_ASSERT(wl->client_id);
fd9fba
+
fd9fba
+    if (pcmk__str_eq(sync_point, PCMK__VALUE_LOCAL, pcmk__str_none)) {
fd9fba
+        wl->sync_point = attrd_sync_point_local;
fd9fba
+    } else if (pcmk__str_eq(sync_point, PCMK__VALUE_CLUSTER, pcmk__str_none)) {
fd9fba
+        wl->sync_point = attrd_sync_point_cluster;
fd9fba
+    } else {
fd9fba
+        free_waitlist_node(wl);
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    wl->ipc_id = request->ipc_id;
fd9fba
+    wl->flags = request->flags;
fd9fba
+
fd9fba
+    crm_debug("Added client %s to waitlist for %s sync point",
fd9fba
+              wl->client_id, sync_point_str(wl->sync_point));
fd9fba
+
fd9fba
+    next_key();
fd9fba
+    pcmk__intkey_table_insert(waitlist, waitlist_client, wl);
fd9fba
+
fd9fba
+    /* And then add the key to the request XML so we can uniquely identify
fd9fba
+     * it when it comes time to issue the ACK.
fd9fba
+     */
fd9fba
+    crm_xml_add_int(request->xml, XML_LRM_ATTR_CALLID, waitlist_client);
fd9fba
+}
fd9fba
+
fd9fba
+void
fd9fba
+attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fd9fba
+{
fd9fba
+    int callid;
fd9fba
+    gpointer value;
fd9fba
+
fd9fba
+    if (waitlist == NULL) {
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    if (crm_element_value_int(xml, XML_LRM_ATTR_CALLID, &callid) == -1) {
fd9fba
+        crm_warn("Could not get callid from request XML");
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    value = pcmk__intkey_table_lookup(waitlist, callid);
fd9fba
+    if (value != NULL) {
fd9fba
+        struct waitlist_node *wl = (struct waitlist_node *) value;
fd9fba
+        pcmk__client_t *client = NULL;
fd9fba
+
fd9fba
+        if (wl->sync_point != sync_point) {
fd9fba
+            return;
fd9fba
+        }
fd9fba
+
fd9fba
+        crm_debug("Alerting client %s for reached %s sync point",
fd9fba
+                  wl->client_id, sync_point_str(wl->sync_point));
fd9fba
+
fd9fba
+        client = pcmk__find_client_by_id(wl->client_id);
fd9fba
+        if (client == NULL) {
fd9fba
+            return;
fd9fba
+        }
fd9fba
+
fd9fba
+        attrd_send_ack(client, wl->ipc_id, wl->flags | crm_ipc_client_response);
fd9fba
+
fd9fba
+        /* And then remove the client so it doesn't get alerted again. */
fd9fba
+        pcmk__intkey_table_remove(waitlist, callid);
fd9fba
+    }
fd9fba
+}
fd9fba
+
fd9fba
 const char *
fd9fba
 attrd_request_sync_point(xmlNode *xml)
fd9fba
 {
fd9fba
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fd9fba
index ff850bb..9dd8320 100644
fd9fba
--- a/daemons/attrd/pacemaker-attrd.h
fd9fba
+++ b/daemons/attrd/pacemaker-attrd.h
fd9fba
@@ -182,6 +182,13 @@ mainloop_timer_t *attrd_add_timer(const char *id, int timeout_ms, attribute_t *a
fd9fba
 void attrd_unregister_handlers(void);
fd9fba
 void attrd_handle_request(pcmk__request_t *request);
fd9fba
 
fd9fba
+enum attrd_sync_point {
fd9fba
+    attrd_sync_point_local,
fd9fba
+    attrd_sync_point_cluster,
fd9fba
+};
fd9fba
+
fd9fba
+void attrd_add_client_to_waitlist(pcmk__request_t *request);
fd9fba
+void attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml);
fd9fba
 const char *attrd_request_sync_point(xmlNode *xml);
fd9fba
 bool attrd_request_has_sync_point(xmlNode *xml);
fd9fba
 
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 59caaf1682191a91d6062358b770f8b9457ba3eb Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Thu, 20 Oct 2022 14:56:58 -0400
fd9fba
Subject: [PATCH 10/26] Feature: daemons: If a client disconnects, remove it
fd9fba
 from the waitlist.
fd9fba
fd9fba
---
fd9fba
 daemons/attrd/attrd_ipc.c       |  5 +++++
fd9fba
 daemons/attrd/attrd_sync.c      | 21 +++++++++++++++++++++
fd9fba
 daemons/attrd/pacemaker-attrd.h |  1 +
fd9fba
 3 files changed, 27 insertions(+)
fd9fba
fd9fba
diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c
fd9fba
index 7e4a1c0..8aa39c2 100644
fd9fba
--- a/daemons/attrd/attrd_ipc.c
fd9fba
+++ b/daemons/attrd/attrd_ipc.c
fd9fba
@@ -438,8 +438,13 @@ attrd_ipc_closed(qb_ipcs_connection_t *c)
fd9fba
         crm_trace("Ignoring request to clean up unknown connection %p", c);
fd9fba
     } else {
fd9fba
         crm_trace("Cleaning up closed client connection %p", c);
fd9fba
+
fd9fba
+        /* Remove the client from the sync point waitlist if it's present. */
fd9fba
+        attrd_remove_client_from_waitlist(client);
fd9fba
+
fd9fba
         pcmk__free_client(client);
fd9fba
     }
fd9fba
+
fd9fba
     return FALSE;
fd9fba
 }
fd9fba
 
fd9fba
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fd9fba
index 2981bd0..7293318 100644
fd9fba
--- a/daemons/attrd/attrd_sync.c
fd9fba
+++ b/daemons/attrd/attrd_sync.c
fd9fba
@@ -112,6 +112,27 @@ attrd_add_client_to_waitlist(pcmk__request_t *request)
fd9fba
     crm_xml_add_int(request->xml, XML_LRM_ATTR_CALLID, waitlist_client);
fd9fba
 }
fd9fba
 
fd9fba
+void
fd9fba
+attrd_remove_client_from_waitlist(pcmk__client_t *client)
fd9fba
+{
fd9fba
+    GHashTableIter iter;
fd9fba
+    gpointer value;
fd9fba
+
fd9fba
+    if (waitlist == NULL) {
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    g_hash_table_iter_init(&iter, waitlist);
fd9fba
+
fd9fba
+    while (g_hash_table_iter_next(&iter, NULL, &value)) {
fd9fba
+        struct waitlist_node *wl = (struct waitlist_node *) value;
fd9fba
+
fd9fba
+        if (wl->client_id == client->id) {
fd9fba
+            g_hash_table_iter_remove(&iter);
fd9fba
+        }
fd9fba
+    }
fd9fba
+}
fd9fba
+
fd9fba
 void
fd9fba
 attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fd9fba
 {
fd9fba
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fd9fba
index 9dd8320..b6ecb75 100644
fd9fba
--- a/daemons/attrd/pacemaker-attrd.h
fd9fba
+++ b/daemons/attrd/pacemaker-attrd.h
fd9fba
@@ -189,6 +189,7 @@ enum attrd_sync_point {
fd9fba
 
fd9fba
 void attrd_add_client_to_waitlist(pcmk__request_t *request);
fd9fba
 void attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml);
fd9fba
+void attrd_remove_client_from_waitlist(pcmk__client_t *client);
fd9fba
 const char *attrd_request_sync_point(xmlNode *xml);
fd9fba
 bool attrd_request_has_sync_point(xmlNode *xml);
fd9fba
 
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From b28042e1d64b48c96dbd9da1e9ee3ff481bbf620 Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Mon, 10 Oct 2022 11:00:20 -0400
fd9fba
Subject: [PATCH 11/26] Feature: daemons: Add support for local sync points on
fd9fba
 clearing failures.
fd9fba
fd9fba
attrd_clear_client_failure just calls attrd_client_update underneath, so
fd9fba
that function will handle all the rest of the sync point functionality
fd9fba
for us.
fd9fba
---
fd9fba
 daemons/attrd/attrd_ipc.c      |  2 --
fd9fba
 daemons/attrd/attrd_messages.c | 19 +++++++++++++++++++
fd9fba
 2 files changed, 19 insertions(+), 2 deletions(-)
fd9fba
fd9fba
diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c
fd9fba
index 8aa39c2..2e614e8 100644
fd9fba
--- a/daemons/attrd/attrd_ipc.c
fd9fba
+++ b/daemons/attrd/attrd_ipc.c
fd9fba
@@ -101,8 +101,6 @@ attrd_client_clear_failure(pcmk__request_t *request)
fd9fba
     xmlNode *xml = request->xml;
fd9fba
     const char *rsc, *op, *interval_spec;
fd9fba
 
fd9fba
-    attrd_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags);
fd9fba
-
fd9fba
     if (minimum_protocol_version >= 2) {
fd9fba
         /* Propagate to all peers (including ourselves).
fd9fba
          * This ends up at attrd_peer_message().
fd9fba
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fd9fba
index c96700f..3ba14a6 100644
fd9fba
--- a/daemons/attrd/attrd_messages.c
fd9fba
+++ b/daemons/attrd/attrd_messages.c
fd9fba
@@ -42,6 +42,25 @@ handle_clear_failure_request(pcmk__request_t *request)
fd9fba
         pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fd9fba
         return NULL;
fd9fba
     } else {
fd9fba
+        if (attrd_request_has_sync_point(request->xml)) {
fd9fba
+            /* If this client supplied a sync point it wants to wait for, add it to
fd9fba
+             * the wait list.  Clients on this list will not receive an ACK until
fd9fba
+             * their sync point is hit which will result in the client stalled there
fd9fba
+             * until it receives a response.
fd9fba
+             *
fd9fba
+             * All other clients will receive the expected response as normal.
fd9fba
+             */
fd9fba
+            attrd_add_client_to_waitlist(request);
fd9fba
+
fd9fba
+        } else {
fd9fba
+            /* If the client doesn't want to wait for a sync point, go ahead and send
fd9fba
+             * the ACK immediately.  Otherwise, we'll send the ACK when the appropriate
fd9fba
+             * sync point is reached.
fd9fba
+             */
fd9fba
+            attrd_send_ack(request->ipc_client, request->ipc_id,
fd9fba
+                           request->ipc_flags);
fd9fba
+        }
fd9fba
+
fd9fba
         return attrd_client_clear_failure(request);
fd9fba
     }
fd9fba
 }
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 291dc3b91e57f2584bbf88cfbe3a360e0332e814 Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Mon, 10 Oct 2022 13:17:24 -0400
fd9fba
Subject: [PATCH 12/26] Refactor: daemons: Free the waitlist on attrd exit.
fd9fba
fd9fba
---
fd9fba
 daemons/attrd/attrd_sync.c      | 11 +++++++++++
fd9fba
 daemons/attrd/attrd_utils.c     |  2 ++
fd9fba
 daemons/attrd/pacemaker-attrd.c |  1 +
fd9fba
 daemons/attrd/pacemaker-attrd.h |  1 +
fd9fba
 4 files changed, 15 insertions(+)
fd9fba
fd9fba
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fd9fba
index 7293318..557e49a 100644
fd9fba
--- a/daemons/attrd/attrd_sync.c
fd9fba
+++ b/daemons/attrd/attrd_sync.c
fd9fba
@@ -112,6 +112,17 @@ attrd_add_client_to_waitlist(pcmk__request_t *request)
fd9fba
     crm_xml_add_int(request->xml, XML_LRM_ATTR_CALLID, waitlist_client);
fd9fba
 }
fd9fba
 
fd9fba
+void
fd9fba
+attrd_free_waitlist(void)
fd9fba
+{
fd9fba
+    if (waitlist == NULL) {
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    g_hash_table_destroy(waitlist);
fd9fba
+    waitlist = NULL;
fd9fba
+}
fd9fba
+
fd9fba
 void
fd9fba
 attrd_remove_client_from_waitlist(pcmk__client_t *client)
fd9fba
 {
fd9fba
diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c
fd9fba
index 6a19009..00b879b 100644
fd9fba
--- a/daemons/attrd/attrd_utils.c
fd9fba
+++ b/daemons/attrd/attrd_utils.c
fd9fba
@@ -93,6 +93,8 @@ attrd_shutdown(int nsig)
fd9fba
     mainloop_destroy_signal(SIGUSR2);
fd9fba
     mainloop_destroy_signal(SIGTRAP);
fd9fba
 
fd9fba
+    attrd_free_waitlist();
fd9fba
+
fd9fba
     if ((mloop == NULL) || !g_main_loop_is_running(mloop)) {
fd9fba
         /* If there's no main loop active, just exit. This should be possible
fd9fba
          * only if we get SIGTERM in brief windows at start-up and shutdown.
fd9fba
diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c
fd9fba
index 2100db4..1336542 100644
fd9fba
--- a/daemons/attrd/pacemaker-attrd.c
fd9fba
+++ b/daemons/attrd/pacemaker-attrd.c
fd9fba
@@ -300,6 +300,7 @@ main(int argc, char **argv)
fd9fba
         attrd_ipc_fini();
fd9fba
         attrd_lrmd_disconnect();
fd9fba
         attrd_cib_disconnect();
fd9fba
+        attrd_free_waitlist();
fd9fba
         g_hash_table_destroy(attributes);
fd9fba
     }
fd9fba
 
fd9fba
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fd9fba
index b6ecb75..537bf85 100644
fd9fba
--- a/daemons/attrd/pacemaker-attrd.h
fd9fba
+++ b/daemons/attrd/pacemaker-attrd.h
fd9fba
@@ -52,6 +52,7 @@ void attrd_run_mainloop(void);
fd9fba
 
fd9fba
 void attrd_set_requesting_shutdown(void);
fd9fba
 void attrd_clear_requesting_shutdown(void);
fd9fba
+void attrd_free_waitlist(void);
fd9fba
 bool attrd_requesting_shutdown(void);
fd9fba
 bool attrd_shutting_down(void);
fd9fba
 void attrd_shutdown(int nsig);
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 7715ce617c520e14687a82e11ff794c93cd7f64a Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Mon, 10 Oct 2022 13:21:16 -0400
fd9fba
Subject: [PATCH 13/26] Feature: includes: Bump CRM_FEATURE_SET for local sync
fd9fba
 points.
fd9fba
fd9fba
---
fd9fba
 include/crm/crm.h | 2 +-
fd9fba
 1 file changed, 1 insertion(+), 1 deletion(-)
fd9fba
fd9fba
diff --git a/include/crm/crm.h b/include/crm/crm.h
fd9fba
index 5710e4b..7c5c602 100644
fd9fba
--- a/include/crm/crm.h
fd9fba
+++ b/include/crm/crm.h
fd9fba
@@ -66,7 +66,7 @@ extern "C" {
fd9fba
  * >=3.0.13: Fail counts include operation name and interval
fd9fba
  * >=3.2.0:  DC supports PCMK_EXEC_INVALID and PCMK_EXEC_NOT_CONNECTED
fd9fba
  */
fd9fba
-#  define CRM_FEATURE_SET		"3.16.1"
fd9fba
+#  define CRM_FEATURE_SET		"3.16.2"
fd9fba
 
fd9fba
 /* Pacemaker's CPG protocols use fixed-width binary fields for the sender and
fd9fba
  * recipient of a CPG message. This imposes an arbitrary limit on cluster node
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From b9054425a76d03f538cd0b3ae27490b1874eee8a Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Fri, 28 Oct 2022 14:23:49 -0400
fd9fba
Subject: [PATCH 14/26] Refactor: daemons: Add comments for previously added
fd9fba
 sync point code.
fd9fba
fd9fba
---
fd9fba
 daemons/attrd/attrd_sync.c | 63 ++++++++++++++++++++++++++++++++++++++
fd9fba
 1 file changed, 63 insertions(+)
fd9fba
fd9fba
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fd9fba
index 557e49a..e9690b5 100644
fd9fba
--- a/daemons/attrd/attrd_sync.c
fd9fba
+++ b/daemons/attrd/attrd_sync.c
fd9fba
@@ -66,6 +66,20 @@ sync_point_str(enum attrd_sync_point sync_point)
fd9fba
     }
fd9fba
 }
fd9fba
 
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief Add a client to the attrd waitlist
fd9fba
+ *
fd9fba
+ * Typically, a client receives an ACK for its XML IPC request immediately.  However,
fd9fba
+ * some clients want to wait until their request has been processed and taken effect.
fd9fba
+ * This is called a sync point.  Any client placed on this waitlist will have its
fd9fba
+ * ACK message delayed until either its requested sync point is hit, or until it
fd9fba
+ * times out.
fd9fba
+ *
fd9fba
+ * The XML IPC request must specify the type of sync point it wants to wait for.
fd9fba
+ *
fd9fba
+ * \param[in,out] request   The request describing the client to place on the waitlist.
fd9fba
+ */
fd9fba
 void
fd9fba
 attrd_add_client_to_waitlist(pcmk__request_t *request)
fd9fba
 {
fd9fba
@@ -112,6 +126,11 @@ attrd_add_client_to_waitlist(pcmk__request_t *request)
fd9fba
     crm_xml_add_int(request->xml, XML_LRM_ATTR_CALLID, waitlist_client);
fd9fba
 }
fd9fba
 
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief Free all memory associated with the waitlist.  This is most typically
fd9fba
+ *        used when attrd shuts down.
fd9fba
+ */
fd9fba
 void
fd9fba
 attrd_free_waitlist(void)
fd9fba
 {
fd9fba
@@ -123,6 +142,13 @@ attrd_free_waitlist(void)
fd9fba
     waitlist = NULL;
fd9fba
 }
fd9fba
 
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief Unconditionally remove a client from the waitlist, such as when the client
fd9fba
+ *        node disconnects from the cluster
fd9fba
+ *
fd9fba
+ * \param[in] client    The client to remove
fd9fba
+ */
fd9fba
 void
fd9fba
 attrd_remove_client_from_waitlist(pcmk__client_t *client)
fd9fba
 {
fd9fba
@@ -144,6 +170,18 @@ attrd_remove_client_from_waitlist(pcmk__client_t *client)
fd9fba
     }
fd9fba
 }
fd9fba
 
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief Send an IPC ACK message to all awaiting clients
fd9fba
+ *
fd9fba
+ * This function will search the waitlist for all clients that are currently awaiting
fd9fba
+ * an ACK indicating their attrd operation is complete.  Only those clients with a
fd9fba
+ * matching sync point type and callid from their original XML IPC request will be
fd9fba
+ * ACKed.  Once they have received an ACK, they will be removed from the waitlist.
fd9fba
+ *
fd9fba
+ * \param[in] sync_point What kind of sync point have we hit?
fd9fba
+ * \param[in] xml        The original XML IPC request.
fd9fba
+ */
fd9fba
 void
fd9fba
 attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fd9fba
 {
fd9fba
@@ -183,6 +221,23 @@ attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fd9fba
     }
fd9fba
 }
fd9fba
 
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief Return the sync point attribute for an IPC request
fd9fba
+ *
fd9fba
+ * This function will check both the top-level element of \p xml for a sync
fd9fba
+ * point attribute, as well as all of its \p op children, if any.  The latter
fd9fba
+ * is useful for newer versions of attrd that can put multiple IPC requests
fd9fba
+ * into a single message.
fd9fba
+ *
fd9fba
+ * \param[in] xml   An XML IPC request
fd9fba
+ *
fd9fba
+ * \note It is assumed that if one child element has a sync point attribute,
fd9fba
+ *       all will have a sync point attribute and they will all be the same
fd9fba
+ *       sync point.  No other configuration is supported.
fd9fba
+ *
fd9fba
+ * \return The sync point attribute of \p xml, or NULL if none.
fd9fba
+ */
fd9fba
 const char *
fd9fba
 attrd_request_sync_point(xmlNode *xml)
fd9fba
 {
fd9fba
@@ -200,6 +255,14 @@ attrd_request_sync_point(xmlNode *xml)
fd9fba
     }
fd9fba
 }
fd9fba
 
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief Does an IPC request contain any sync point attribute?
fd9fba
+ *
fd9fba
+ * \param[in] xml   An XML IPC request
fd9fba
+ *
fd9fba
+ * \return true if there's a sync point attribute, false otherwise
fd9fba
+ */
fd9fba
 bool
fd9fba
 attrd_request_has_sync_point(xmlNode *xml)
fd9fba
 {
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 64219fb7075ee58d29f94f077a3b8f94174bb32a Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Wed, 26 Oct 2022 12:43:05 -0400
fd9fba
Subject: [PATCH 15/26] Feature: tools: Add --wait=cluster option to
fd9fba
 attrd_updater.
fd9fba
fd9fba
---
fd9fba
 tools/attrd_updater.c | 10 ++++++++--
fd9fba
 1 file changed, 8 insertions(+), 2 deletions(-)
fd9fba
fd9fba
diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c
fd9fba
index c4779a6..3cd766d 100644
fd9fba
--- a/tools/attrd_updater.c
fd9fba
+++ b/tools/attrd_updater.c
fd9fba
@@ -106,6 +106,10 @@ wait_cb (const gchar *option_name, const gchar *optarg, gpointer data, GError **
fd9fba
         pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
fd9fba
         pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local);
fd9fba
         return TRUE;
fd9fba
+    } else if (pcmk__str_eq(optarg, PCMK__VALUE_CLUSTER, pcmk__str_none)) {
fd9fba
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
fd9fba
+        pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_sync_cluster);
fd9fba
+        return TRUE;
fd9fba
     } else {
fd9fba
         g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_USAGE,
fd9fba
                     "--wait= must be one of 'no', 'local', 'cluster'");
fd9fba
@@ -193,10 +197,12 @@ static GOptionEntry addl_entries[] = {
fd9fba
 
fd9fba
     { "wait", 'W', 0, G_OPTION_ARG_CALLBACK, wait_cb,
fd9fba
       "Wait for some event to occur before returning.  Values are 'no' (wait\n"
fd9fba
-      INDENT "only for the attribute daemon to acknowledge the request) or\n"
fd9fba
+      INDENT "only for the attribute daemon to acknowledge the request),\n"
fd9fba
       INDENT "'local' (wait until the change has propagated to where a local\n"
fd9fba
       INDENT "query will return the request value, or the value set by a\n"
fd9fba
-      INDENT "later request).  Default is 'no'.",
fd9fba
+      INDENT "later request), or 'cluster' (wait until the change has propagated\n"
fd9fba
+      INDENT "to where a query anywhere on the cluster will return the requested\n"
fd9fba
+      INDENT "value, or the value set by a later request).  Default is 'no'.",
fd9fba
       "UNTIL" },
fd9fba
 
fd9fba
     { NULL }
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 1bc5511fadf6ad670508bd3a2a55129bde16f774 Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Fri, 16 Sep 2022 14:55:06 -0400
fd9fba
Subject: [PATCH 16/26] Refactor: daemons: Add a confirm= attribute to attrd
fd9fba
 messages.
fd9fba
fd9fba
This allows informing the originator of a message that the message has
fd9fba
been received and processed.  As yet, there is no mechanism for handling
fd9fba
and returning the confirmation, only for requesting it.
fd9fba
---
fd9fba
 daemons/attrd/attrd_corosync.c  |  6 +++---
fd9fba
 daemons/attrd/attrd_ipc.c       | 26 +++++++++++++++++++++-----
fd9fba
 daemons/attrd/attrd_messages.c  | 11 +++++++++--
fd9fba
 daemons/attrd/pacemaker-attrd.h |  7 ++++---
fd9fba
 include/crm_internal.h          |  1 +
fd9fba
 5 files changed, 38 insertions(+), 13 deletions(-)
fd9fba
fd9fba
diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
fd9fba
index 4337280..e86ca07 100644
fd9fba
--- a/daemons/attrd/attrd_corosync.c
fd9fba
+++ b/daemons/attrd/attrd_corosync.c
fd9fba
@@ -124,7 +124,7 @@ broadcast_local_value(const attribute_t *a)
fd9fba
 
fd9fba
     crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
fd9fba
     attrd_add_value_xml(sync, a, v, false);
fd9fba
-    attrd_send_message(NULL, sync);
fd9fba
+    attrd_send_message(NULL, sync, false);
fd9fba
     free_xml(sync);
fd9fba
     return v;
fd9fba
 }
fd9fba
@@ -387,7 +387,7 @@ broadcast_unseen_local_values(void)
fd9fba
 
fd9fba
     if (sync != NULL) {
fd9fba
         crm_debug("Broadcasting local-only values");
fd9fba
-        attrd_send_message(NULL, sync);
fd9fba
+        attrd_send_message(NULL, sync, false);
fd9fba
         free_xml(sync);
fd9fba
     }
fd9fba
 }
fd9fba
@@ -539,7 +539,7 @@ attrd_peer_sync(crm_node_t *peer, xmlNode *xml)
fd9fba
     }
fd9fba
 
fd9fba
     crm_debug("Syncing values to %s", peer?peer->uname:"everyone");
fd9fba
-    attrd_send_message(peer, sync);
fd9fba
+    attrd_send_message(peer, sync, false);
fd9fba
     free_xml(sync);
fd9fba
 }
fd9fba
 
fd9fba
diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c
fd9fba
index 2e614e8..0fc5e93 100644
fd9fba
--- a/daemons/attrd/attrd_ipc.c
fd9fba
+++ b/daemons/attrd/attrd_ipc.c
fd9fba
@@ -105,7 +105,7 @@ attrd_client_clear_failure(pcmk__request_t *request)
fd9fba
         /* Propagate to all peers (including ourselves).
fd9fba
          * This ends up at attrd_peer_message().
fd9fba
          */
fd9fba
-        attrd_send_message(NULL, xml);
fd9fba
+        attrd_send_message(NULL, xml, false);
fd9fba
         pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fd9fba
         return NULL;
fd9fba
     }
fd9fba
@@ -184,7 +184,7 @@ attrd_client_peer_remove(pcmk__request_t *request)
fd9fba
     if (host) {
fd9fba
         crm_info("Client %s is requesting all values for %s be removed",
fd9fba
                  pcmk__client_name(request->ipc_client), host);
fd9fba
-        attrd_send_message(NULL, xml); /* ends up at attrd_peer_message() */
fd9fba
+        attrd_send_message(NULL, xml, false); /* ends up at attrd_peer_message() */
fd9fba
         free(host_alloc);
fd9fba
     } else {
fd9fba
         crm_info("Ignoring request by client %s to remove all peer values without specifying peer",
fd9fba
@@ -314,7 +314,7 @@ attrd_client_update(pcmk__request_t *request)
fd9fba
                 }
fd9fba
             }
fd9fba
 
fd9fba
-            attrd_send_message(NULL, xml);
fd9fba
+            attrd_send_message(NULL, xml, false);
fd9fba
             pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fd9fba
 
fd9fba
         } else {
fd9fba
@@ -358,7 +358,7 @@ attrd_client_update(pcmk__request_t *request)
fd9fba
                 if (status == 0) {
fd9fba
                     crm_trace("Matched %s with %s", attr, regex);
fd9fba
                     crm_xml_add(xml, PCMK__XA_ATTR_NAME, attr);
fd9fba
-                    attrd_send_message(NULL, xml);
fd9fba
+                    attrd_send_message(NULL, xml, false);
fd9fba
                 }
fd9fba
             }
fd9fba
 
fd9fba
@@ -388,7 +388,23 @@ attrd_client_update(pcmk__request_t *request)
fd9fba
     crm_debug("Broadcasting %s[%s]=%s%s", attr, crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME),
fd9fba
               value, (attrd_election_won()? " (writer)" : ""));
fd9fba
 
fd9fba
-    attrd_send_message(NULL, xml); /* ends up at attrd_peer_message() */
fd9fba
+    if (pcmk__str_eq(attrd_request_sync_point(xml), PCMK__VALUE_CLUSTER, pcmk__str_none)) {
fd9fba
+        /* The client is waiting on the cluster-wide sync point.  In this case,
fd9fba
+         * the response ACK is not sent until this attrd broadcasts the update
fd9fba
+         * and receives its own confirmation back from all peers.
fd9fba
+         */
fd9fba
+        attrd_send_message(NULL, xml, true); /* ends up at attrd_peer_message() */
fd9fba
+
fd9fba
+    } else {
fd9fba
+        /* The client is either waiting on the local sync point or was not
fd9fba
+         * waiting on any sync point at all.  For the local sync point, the
fd9fba
+         * response ACK is sent in attrd_peer_update.  For clients not
fd9fba
+         * waiting on any sync point, the response ACK is sent in
fd9fba
+         * handle_update_request immediately before this function was called.
fd9fba
+         */
fd9fba
+        attrd_send_message(NULL, xml, false); /* ends up at attrd_peer_message() */
fd9fba
+    }
fd9fba
+
fd9fba
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fd9fba
     return NULL;
fd9fba
 }
fd9fba
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fd9fba
index 3ba14a6..78df0d0 100644
fd9fba
--- a/daemons/attrd/attrd_messages.c
fd9fba
+++ b/daemons/attrd/attrd_messages.c
fd9fba
@@ -279,16 +279,23 @@ attrd_broadcast_protocol(void)
fd9fba
     crm_debug("Broadcasting attrd protocol version %s for node %s",
fd9fba
               ATTRD_PROTOCOL_VERSION, attrd_cluster->uname);
fd9fba
 
fd9fba
-    attrd_send_message(NULL, attrd_op); /* ends up at attrd_peer_message() */
fd9fba
+    attrd_send_message(NULL, attrd_op, false); /* ends up at attrd_peer_message() */
fd9fba
 
fd9fba
     free_xml(attrd_op);
fd9fba
 }
fd9fba
 
fd9fba
 gboolean
fd9fba
-attrd_send_message(crm_node_t * node, xmlNode * data)
fd9fba
+attrd_send_message(crm_node_t *node, xmlNode *data, bool confirm)
fd9fba
 {
fd9fba
     crm_xml_add(data, F_TYPE, T_ATTRD);
fd9fba
     crm_xml_add(data, PCMK__XA_ATTR_VERSION, ATTRD_PROTOCOL_VERSION);
fd9fba
+
fd9fba
+    /* Request a confirmation from the destination peer node (which could
fd9fba
+     * be all if node is NULL) that the message has been received and
fd9fba
+     * acted upon.
fd9fba
+     */
fd9fba
+    pcmk__xe_set_bool_attr(data, PCMK__XA_CONFIRM, confirm);
fd9fba
+
fd9fba
     attrd_xml_add_writer(data);
fd9fba
     return send_cluster_message(node, crm_msg_attrd, data, TRUE);
fd9fba
 }
fd9fba
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fd9fba
index 537bf85..25f7c8a 100644
fd9fba
--- a/daemons/attrd/pacemaker-attrd.h
fd9fba
+++ b/daemons/attrd/pacemaker-attrd.h
fd9fba
@@ -39,10 +39,11 @@
fd9fba
  *                      PCMK__ATTRD_CMD_UPDATE_DELAY
fd9fba
  *     2       1.1.17   PCMK__ATTRD_CMD_CLEAR_FAILURE
fd9fba
  *     3       2.1.1    PCMK__ATTRD_CMD_SYNC_RESPONSE indicates remote nodes
fd9fba
- *     4       2.2.0    Multiple attributes can be updated in a single IPC
fd9fba
+ *     4       2.1.5    Multiple attributes can be updated in a single IPC
fd9fba
  *                      message
fd9fba
+ *     5       2.1.5    Peers can request confirmation of a sent message
fd9fba
  */
fd9fba
-#define ATTRD_PROTOCOL_VERSION "4"
fd9fba
+#define ATTRD_PROTOCOL_VERSION "5"
fd9fba
 
fd9fba
 #define attrd_send_ack(client, id, flags) \
fd9fba
     pcmk__ipc_send_ack((client), (id), (flags), "ack", ATTRD_PROTOCOL_VERSION, CRM_EX_INDETERMINATE)
fd9fba
@@ -162,7 +163,7 @@ xmlNode *attrd_client_clear_failure(pcmk__request_t *request);
fd9fba
 xmlNode *attrd_client_update(pcmk__request_t *request);
fd9fba
 xmlNode *attrd_client_refresh(pcmk__request_t *request);
fd9fba
 xmlNode *attrd_client_query(pcmk__request_t *request);
fd9fba
-gboolean attrd_send_message(crm_node_t * node, xmlNode * data);
fd9fba
+gboolean attrd_send_message(crm_node_t *node, xmlNode *data, bool confirm);
fd9fba
 
fd9fba
 xmlNode *attrd_add_value_xml(xmlNode *parent, const attribute_t *a,
fd9fba
                              const attribute_value_t *v, bool force_write);
fd9fba
diff --git a/include/crm_internal.h b/include/crm_internal.h
fd9fba
index 08193c3..63a1726 100644
fd9fba
--- a/include/crm_internal.h
fd9fba
+++ b/include/crm_internal.h
fd9fba
@@ -79,6 +79,7 @@
fd9fba
 #define PCMK__XA_ATTR_WRITER            "attr_writer"
fd9fba
 #define PCMK__XA_CONFIG_ERRORS          "config-errors"
fd9fba
 #define PCMK__XA_CONFIG_WARNINGS        "config-warnings"
fd9fba
+#define PCMK__XA_CONFIRM                "confirm"
fd9fba
 #define PCMK__XA_GRAPH_ERRORS           "graph-errors"
fd9fba
 #define PCMK__XA_GRAPH_WARNINGS         "graph-warnings"
fd9fba
 #define PCMK__XA_MODE                   "mode"
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 6f389038fc0b11f6291c022c99f188666c65f530 Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Wed, 26 Oct 2022 14:44:42 -0400
fd9fba
Subject: [PATCH 17/26] Feature: daemons: Respond to received attrd
fd9fba
 confirmation requests.
fd9fba
fd9fba
On the receiving peer side, if the XML request contains confirm="true",
fd9fba
construct a confirmation message after handling the request completes
fd9fba
and send it back to the originating peer.
fd9fba
fd9fba
On the originating peer side, add a skeleton handler for confirmation
fd9fba
messages.  This does nothing at the moment except log it.
fd9fba
---
fd9fba
 daemons/attrd/attrd_corosync.c | 38 ++++++++++++++++++++++++++++++++++
fd9fba
 daemons/attrd/attrd_messages.c | 13 ++++++++++++
fd9fba
 include/crm_internal.h         |  1 +
fd9fba
 3 files changed, 52 insertions(+)
fd9fba
fd9fba
diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
fd9fba
index e86ca07..1245d9c 100644
fd9fba
--- a/daemons/attrd/attrd_corosync.c
fd9fba
+++ b/daemons/attrd/attrd_corosync.c
fd9fba
@@ -25,6 +25,19 @@
fd9fba
 
fd9fba
 extern crm_exit_t attrd_exit_status;
fd9fba
 
fd9fba
+static xmlNode *
fd9fba
+attrd_confirmation(int callid)
fd9fba
+{
fd9fba
+    xmlNode *node = create_xml_node(NULL, __func__);
fd9fba
+
fd9fba
+    crm_xml_add(node, F_TYPE, T_ATTRD);
fd9fba
+    crm_xml_add(node, F_ORIG, get_local_node_name());
fd9fba
+    crm_xml_add(node, PCMK__XA_TASK, PCMK__ATTRD_CMD_CONFIRM);
fd9fba
+    crm_xml_add_int(node, XML_LRM_ATTR_CALLID, callid);
fd9fba
+
fd9fba
+    return node;
fd9fba
+}
fd9fba
+
fd9fba
 static void
fd9fba
 attrd_peer_message(crm_node_t *peer, xmlNode *xml)
fd9fba
 {
fd9fba
@@ -57,6 +70,31 @@ attrd_peer_message(crm_node_t *peer, xmlNode *xml)
fd9fba
         CRM_CHECK(request.op != NULL, return);
fd9fba
 
fd9fba
         attrd_handle_request(&request);
fd9fba
+
fd9fba
+        /* Having finished handling the request, check to see if the originating
fd9fba
+         * peer requested confirmation.  If so, send that confirmation back now.
fd9fba
+         */
fd9fba
+        if (pcmk__xe_attr_is_true(xml, PCMK__XA_CONFIRM)) {
fd9fba
+            int callid = 0;
fd9fba
+            xmlNode *reply = NULL;
fd9fba
+
fd9fba
+            /* Add the confirmation ID for the message we are confirming to the
fd9fba
+             * response so the originating peer knows what they're a confirmation
fd9fba
+             * for.
fd9fba
+             */
fd9fba
+            crm_element_value_int(xml, XML_LRM_ATTR_CALLID, &callid);
fd9fba
+            reply = attrd_confirmation(callid);
fd9fba
+
fd9fba
+            /* And then send the confirmation back to the originating peer.  This
fd9fba
+             * ends up right back in this same function (attrd_peer_message) on the
fd9fba
+             * peer where it will have to do something with a PCMK__XA_CONFIRM type
fd9fba
+             * message.
fd9fba
+             */
fd9fba
+            crm_debug("Sending %s a confirmation", peer->uname);
fd9fba
+            attrd_send_message(peer, reply, false);
fd9fba
+            free_xml(reply);
fd9fba
+        }
fd9fba
+
fd9fba
         pcmk__reset_request(&request);
fd9fba
     }
fd9fba
 }
fd9fba
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fd9fba
index 78df0d0..9c792b2 100644
fd9fba
--- a/daemons/attrd/attrd_messages.c
fd9fba
+++ b/daemons/attrd/attrd_messages.c
fd9fba
@@ -65,6 +65,18 @@ handle_clear_failure_request(pcmk__request_t *request)
fd9fba
     }
fd9fba
 }
fd9fba
 
fd9fba
+static xmlNode *
fd9fba
+handle_confirm_request(pcmk__request_t *request)
fd9fba
+{
fd9fba
+    if (request->peer != NULL) {
fd9fba
+        crm_debug("Received confirmation from %s", request->peer);
fd9fba
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fd9fba
+        return NULL;
fd9fba
+    } else {
fd9fba
+        return handle_unknown_request(request);
fd9fba
+    }
fd9fba
+}
fd9fba
+
fd9fba
 static xmlNode *
fd9fba
 handle_flush_request(pcmk__request_t *request)
fd9fba
 {
fd9fba
@@ -190,6 +202,7 @@ attrd_register_handlers(void)
fd9fba
 {
fd9fba
     pcmk__server_command_t handlers[] = {
fd9fba
         { PCMK__ATTRD_CMD_CLEAR_FAILURE, handle_clear_failure_request },
fd9fba
+        { PCMK__ATTRD_CMD_CONFIRM, handle_confirm_request },
fd9fba
         { PCMK__ATTRD_CMD_FLUSH, handle_flush_request },
fd9fba
         { PCMK__ATTRD_CMD_PEER_REMOVE, handle_remove_request },
fd9fba
         { PCMK__ATTRD_CMD_QUERY, handle_query_request },
fd9fba
diff --git a/include/crm_internal.h b/include/crm_internal.h
fd9fba
index 63a1726..f60e7b4 100644
fd9fba
--- a/include/crm_internal.h
fd9fba
+++ b/include/crm_internal.h
fd9fba
@@ -108,6 +108,7 @@
fd9fba
 #define PCMK__ATTRD_CMD_SYNC            "sync"
fd9fba
 #define PCMK__ATTRD_CMD_SYNC_RESPONSE   "sync-response"
fd9fba
 #define PCMK__ATTRD_CMD_CLEAR_FAILURE   "clear-failure"
fd9fba
+#define PCMK__ATTRD_CMD_CONFIRM         "confirm"
fd9fba
 
fd9fba
 #define PCMK__CONTROLD_CMD_NODES        "list-nodes"
fd9fba
 
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From dfb730e9ced9dc75886fda9452c584860573fe30 Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Wed, 26 Oct 2022 15:58:00 -0400
fd9fba
Subject: [PATCH 18/26] Feature: daemons: Keep track of #attrd-protocol from
fd9fba
 each peer.
fd9fba
fd9fba
This information can be used in the future when dealing with
fd9fba
cluster-wide sync points to know which peers we are waiting on a reply
fd9fba
from.
fd9fba
---
fd9fba
 daemons/attrd/attrd_corosync.c  |  3 +-
fd9fba
 daemons/attrd/attrd_utils.c     | 60 ++++++++++++++++++++++++++++++---
fd9fba
 daemons/attrd/pacemaker-attrd.h |  4 ++-
fd9fba
 3 files changed, 60 insertions(+), 7 deletions(-)
fd9fba
fd9fba
diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
fd9fba
index 1245d9c..6f88ab6 100644
fd9fba
--- a/daemons/attrd/attrd_corosync.c
fd9fba
+++ b/daemons/attrd/attrd_corosync.c
fd9fba
@@ -268,6 +268,7 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da
fd9fba
     // Remove votes from cluster nodes that leave, in case election in progress
fd9fba
     if (gone && !is_remote) {
fd9fba
         attrd_remove_voter(peer);
fd9fba
+        attrd_remove_peer_protocol_ver(peer->uname);
fd9fba
 
fd9fba
     // Ensure remote nodes that come up are in the remote node cache
fd9fba
     } else if (!gone && is_remote) {
fd9fba
@@ -395,7 +396,7 @@ attrd_peer_update_one(const crm_node_t *peer, xmlNode *xml, bool filter)
fd9fba
      * version, check to see if it's a new minimum version.
fd9fba
      */
fd9fba
     if (pcmk__str_eq(attr, CRM_ATTR_PROTOCOL, pcmk__str_none)) {
fd9fba
-        attrd_update_minimum_protocol_ver(value);
fd9fba
+        attrd_update_minimum_protocol_ver(peer->uname, value);
fd9fba
     }
fd9fba
 }
fd9fba
 
fd9fba
diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c
fd9fba
index 00b879b..421faed 100644
fd9fba
--- a/daemons/attrd/attrd_utils.c
fd9fba
+++ b/daemons/attrd/attrd_utils.c
fd9fba
@@ -29,6 +29,11 @@ static bool requesting_shutdown = false;
fd9fba
 static bool shutting_down = false;
fd9fba
 static GMainLoop *mloop = NULL;
fd9fba
 
fd9fba
+/* A hash table storing information on the protocol version of each peer attrd.
fd9fba
+ * The key is the peer's uname, and the value is the protocol version number.
fd9fba
+ */
fd9fba
+GHashTable *peer_protocol_vers = NULL;
fd9fba
+
fd9fba
 /*!
fd9fba
  * \internal
fd9fba
  * \brief  Set requesting_shutdown state
fd9fba
@@ -94,6 +99,10 @@ attrd_shutdown(int nsig)
fd9fba
     mainloop_destroy_signal(SIGTRAP);
fd9fba
 
fd9fba
     attrd_free_waitlist();
fd9fba
+    if (peer_protocol_vers != NULL) {
fd9fba
+        g_hash_table_destroy(peer_protocol_vers);
fd9fba
+        peer_protocol_vers = NULL;
fd9fba
+    }
fd9fba
 
fd9fba
     if ((mloop == NULL) || !g_main_loop_is_running(mloop)) {
fd9fba
         /* If there's no main loop active, just exit. This should be possible
fd9fba
@@ -273,16 +282,57 @@ attrd_free_attribute(gpointer data)
fd9fba
     }
fd9fba
 }
fd9fba
 
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief When a peer node leaves the cluster, stop tracking its protocol version.
fd9fba
+ *
fd9fba
+ * \param[in] host  The peer node's uname to be removed
fd9fba
+ */
fd9fba
+void
fd9fba
+attrd_remove_peer_protocol_ver(const char *host)
fd9fba
+{
fd9fba
+    if (peer_protocol_vers != NULL) {
fd9fba
+        g_hash_table_remove(peer_protocol_vers, host);
fd9fba
+    }
fd9fba
+}
fd9fba
+
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief When a peer node broadcasts a message with its protocol version, keep
fd9fba
+ *        track of that information.
fd9fba
+ *
fd9fba
+ * We keep track of each peer's protocol version so we know which peers to
fd9fba
+ * expect confirmation messages from when handling cluster-wide sync points.
fd9fba
+ * We additionally keep track of the lowest protocol version supported by all
fd9fba
+ * peers so we know when we can send IPC messages containing more than one
fd9fba
+ * request.
fd9fba
+ *
fd9fba
+ * \param[in] host  The peer node's uname to be tracked
fd9fba
+ * \param[in] value The peer node's protocol version
fd9fba
+ */
fd9fba
 void
fd9fba
-attrd_update_minimum_protocol_ver(const char *value)
fd9fba
+attrd_update_minimum_protocol_ver(const char *host, const char *value)
fd9fba
 {
fd9fba
     int ver;
fd9fba
 
fd9fba
+    if (peer_protocol_vers == NULL) {
fd9fba
+        peer_protocol_vers = pcmk__strkey_table(free, NULL);
fd9fba
+    }
fd9fba
+
fd9fba
     pcmk__scan_min_int(value, &ver, 0);
fd9fba
 
fd9fba
-    if (ver > 0 && (minimum_protocol_version == -1 || ver < minimum_protocol_version)) {
fd9fba
-        minimum_protocol_version = ver;
fd9fba
-        crm_trace("Set minimum attrd protocol version to %d",
fd9fba
-                  minimum_protocol_version);
fd9fba
+    if (ver > 0) {
fd9fba
+        char *host_name = strdup(host);
fd9fba
+
fd9fba
+        /* Record the peer attrd's protocol version. */
fd9fba
+        CRM_ASSERT(host_name != NULL);
fd9fba
+        g_hash_table_insert(peer_protocol_vers, host_name, GINT_TO_POINTER(ver));
fd9fba
+
fd9fba
+        /* If the protocol version is a new minimum, record it as such. */
fd9fba
+        if (minimum_protocol_version == -1 || ver < minimum_protocol_version) {
fd9fba
+            minimum_protocol_version = ver;
fd9fba
+            crm_trace("Set minimum attrd protocol version to %d",
fd9fba
+                      minimum_protocol_version);
fd9fba
+        }
fd9fba
     }
fd9fba
 }
fd9fba
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fd9fba
index 25f7c8a..302ef63 100644
fd9fba
--- a/daemons/attrd/pacemaker-attrd.h
fd9fba
+++ b/daemons/attrd/pacemaker-attrd.h
fd9fba
@@ -145,6 +145,7 @@ typedef struct attribute_value_s {
fd9fba
 
fd9fba
 extern crm_cluster_t *attrd_cluster;
fd9fba
 extern GHashTable *attributes;
fd9fba
+extern GHashTable *peer_protocol_vers;
fd9fba
 
fd9fba
 #define CIB_OP_TIMEOUT_S 120
fd9fba
 
fd9fba
@@ -177,7 +178,8 @@ void attrd_write_attributes(bool all, bool ignore_delay);
fd9fba
 void attrd_write_or_elect_attribute(attribute_t *a);
fd9fba
 
fd9fba
 extern int minimum_protocol_version;
fd9fba
-void attrd_update_minimum_protocol_ver(const char *value);
fd9fba
+void attrd_remove_peer_protocol_ver(const char *host);
fd9fba
+void attrd_update_minimum_protocol_ver(const char *host, const char *value);
fd9fba
 
fd9fba
 mainloop_timer_t *attrd_add_timer(const char *id, int timeout_ms, attribute_t *attr);
fd9fba
 
fd9fba
-- 
fd9fba
2.31.1
fd9fba
fd9fba
From 945f0fe51d3bf69c2cb1258b394f2f11b8996525 Mon Sep 17 00:00:00 2001
fd9fba
From: Chris Lumens <clumens@redhat.com>
fd9fba
Date: Thu, 27 Oct 2022 14:42:59 -0400
fd9fba
Subject: [PATCH 19/26] Feature: daemons: Handle cluster-wide sync points in
fd9fba
 attrd.
fd9fba
fd9fba
When an attrd receives an IPC request to update some value, record the
fd9fba
protocol versions of all peer attrds.  Additionally register a function
fd9fba
that will be called when all confirmations are received.
fd9fba
fd9fba
The originating IPC cilent (attrd_updater for instance) will sit there
fd9fba
waiting for an ACK until its timeout is hit.
fd9fba
fd9fba
As each confirmation message comes back to attrd, mark it off the list
fd9fba
of peers we are waiting on.  When no more peers are expected, call the
fd9fba
previously registered function.
fd9fba
fd9fba
For attribute updates, this function just sends an ack back to
fd9fba
attrd_updater.
fd9fba
fd9fba
Fixes T35
fd9fba
---
fd9fba
 daemons/attrd/attrd_corosync.c  |   1 +
fd9fba
 daemons/attrd/attrd_ipc.c       |   4 +
fd9fba
 daemons/attrd/attrd_messages.c  |  10 ++
fd9fba
 daemons/attrd/attrd_sync.c      | 260 +++++++++++++++++++++++++++++++-
fd9fba
 daemons/attrd/attrd_utils.c     |   2 +
fd9fba
 daemons/attrd/pacemaker-attrd.h |   8 +
fd9fba
 6 files changed, 281 insertions(+), 4 deletions(-)
fd9fba
fd9fba
diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
fd9fba
index 6f88ab6..37701aa 100644
fd9fba
--- a/daemons/attrd/attrd_corosync.c
fd9fba
+++ b/daemons/attrd/attrd_corosync.c
fd9fba
@@ -269,6 +269,7 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da
fd9fba
     if (gone && !is_remote) {
fd9fba
         attrd_remove_voter(peer);
fd9fba
         attrd_remove_peer_protocol_ver(peer->uname);
fd9fba
+        attrd_do_not_expect_from_peer(peer->uname);
fd9fba
 
fd9fba
     // Ensure remote nodes that come up are in the remote node cache
fd9fba
     } else if (!gone && is_remote) {
fd9fba
diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c
fd9fba
index 0fc5e93..c70aa1b 100644
fd9fba
--- a/daemons/attrd/attrd_ipc.c
fd9fba
+++ b/daemons/attrd/attrd_ipc.c
fd9fba
@@ -393,6 +393,7 @@ attrd_client_update(pcmk__request_t *request)
fd9fba
          * the response ACK is not sent until this attrd broadcasts the update
fd9fba
          * and receives its own confirmation back from all peers.
fd9fba
          */
fd9fba
+        attrd_expect_confirmations(request, attrd_cluster_sync_point_update);
fd9fba
         attrd_send_message(NULL, xml, true); /* ends up at attrd_peer_message() */
fd9fba
 
fd9fba
     } else {
fd9fba
@@ -456,6 +457,9 @@ attrd_ipc_closed(qb_ipcs_connection_t *c)
fd9fba
         /* Remove the client from the sync point waitlist if it's present. */
fd9fba
         attrd_remove_client_from_waitlist(client);
fd9fba
 
fd9fba
+        /* And no longer wait for confirmations from any peers. */
fd9fba
+        attrd_do_not_wait_for_client(client);
fd9fba
+
fd9fba
         pcmk__free_client(client);
fd9fba
     }
fd9fba
 
fd9fba
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fd9fba
index 9c792b2..f7b9c7c 100644
fd9fba
--- a/daemons/attrd/attrd_messages.c
fd9fba
+++ b/daemons/attrd/attrd_messages.c
fd9fba
@@ -69,7 +69,17 @@ static xmlNode *
fd9fba
 handle_confirm_request(pcmk__request_t *request)
fd9fba
 {
fd9fba
     if (request->peer != NULL) {
fd9fba
+        int callid;
fd9fba
+
fd9fba
         crm_debug("Received confirmation from %s", request->peer);
fd9fba
+
fd9fba
+        if (crm_element_value_int(request->xml, XML_LRM_ATTR_CALLID, &callid) == -1) {
fd9fba
+            pcmk__set_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
fd9fba
+                             "Could not get callid from XML");
fd9fba
+        } else {
fd9fba
+            attrd_handle_confirmation(callid, request->peer);
fd9fba
+        }
fd9fba
+
fd9fba
         pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fd9fba
         return NULL;
fd9fba
     } else {
fd9fba
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fd9fba
index e9690b5..d3d7108 100644
fd9fba
--- a/daemons/attrd/attrd_sync.c
fd9fba
+++ b/daemons/attrd/attrd_sync.c
fd9fba
@@ -34,6 +34,51 @@ struct waitlist_node {
fd9fba
     uint32_t flags;
fd9fba
 };
fd9fba
 
fd9fba
+/* A hash table storing information on in-progress IPC requests that are awaiting
fd9fba
+ * confirmations.  These requests are currently being processed by peer attrds and
fd9fba
+ * we are waiting to receive confirmation messages from each peer indicating that
fd9fba
+ * processing is complete.
fd9fba
+ *
fd9fba
+ * Multiple requests could be waiting on confirmations at the same time.
fd9fba
+ *
fd9fba
+ * The key is the unique callid for the IPC request, and the value is a
fd9fba
+ * confirmation_action struct.
fd9fba
+ */
fd9fba
+static GHashTable *expected_confirmations = NULL;
fd9fba
+
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief A structure describing a single IPC request that is awaiting confirmations
fd9fba
+ */
fd9fba
+struct confirmation_action {
fd9fba
+    /*!
fd9fba
+     * \brief A list of peer attrds that we are waiting to receive confirmation
fd9fba
+     *        messages from
fd9fba
+     *
fd9fba
+     * This list is dynamic - as confirmations arrive from peer attrds, they will
fd9fba
+     * be removed from this list.  When the list is empty, all peers have processed
fd9fba
+     * the request and the associated confirmation action will be taken.
fd9fba
+     */
fd9fba
+    GList *respondents;
fd9fba
+
fd9fba
+    /*!
fd9fba
+     * \brief A function to run when all confirmations have been received
fd9fba
+     */
fd9fba
+    attrd_confirmation_action_fn fn;
fd9fba
+
fd9fba
+    /*!
fd9fba
+     * \brief Information required to construct and send a reply to the client
fd9fba
+     */
fd9fba
+    char *client_id;
fd9fba
+    uint32_t ipc_id;
fd9fba
+    uint32_t flags;
fd9fba
+
fd9fba
+    /*!
fd9fba
+     * \brief The XML request containing the callid associated with this action
fd9fba
+     */
fd9fba
+    void *xml;
fd9fba
+};
fd9fba
+
fd9fba
 static void
fd9fba
 next_key(void)
fd9fba
 {
fd9fba
@@ -114,12 +159,13 @@ attrd_add_client_to_waitlist(pcmk__request_t *request)
fd9fba
     wl->ipc_id = request->ipc_id;
fd9fba
     wl->flags = request->flags;
fd9fba
 
fd9fba
-    crm_debug("Added client %s to waitlist for %s sync point",
fd9fba
-              wl->client_id, sync_point_str(wl->sync_point));
fd9fba
-
fd9fba
     next_key();
fd9fba
     pcmk__intkey_table_insert(waitlist, waitlist_client, wl);
fd9fba
 
fd9fba
+    crm_trace("Added client %s to waitlist for %s sync point",
fd9fba
+              wl->client_id, sync_point_str(wl->sync_point));
fd9fba
+    crm_trace("%d clients now on waitlist", g_hash_table_size(waitlist));
fd9fba
+
fd9fba
     /* And then add the key to the request XML so we can uniquely identify
fd9fba
      * it when it comes time to issue the ACK.
fd9fba
      */
fd9fba
@@ -166,6 +212,7 @@ attrd_remove_client_from_waitlist(pcmk__client_t *client)
fd9fba
 
fd9fba
         if (wl->client_id == client->id) {
fd9fba
             g_hash_table_iter_remove(&iter);
fd9fba
+            crm_trace("%d clients now on waitlist", g_hash_table_size(waitlist));
fd9fba
         }
fd9fba
     }
fd9fba
 }
fd9fba
@@ -206,7 +253,7 @@ attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fd9fba
             return;
fd9fba
         }
fd9fba
 
fd9fba
-        crm_debug("Alerting client %s for reached %s sync point",
fd9fba
+        crm_trace("Alerting client %s for reached %s sync point",
fd9fba
                   wl->client_id, sync_point_str(wl->sync_point));
fd9fba
 
fd9fba
         client = pcmk__find_client_by_id(wl->client_id);
fd9fba
@@ -218,9 +265,28 @@ attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fd9fba
 
fd9fba
         /* And then remove the client so it doesn't get alerted again. */
fd9fba
         pcmk__intkey_table_remove(waitlist, callid);
fd9fba
+
fd9fba
+        crm_trace("%d clients now on waitlist", g_hash_table_size(waitlist));
fd9fba
     }
fd9fba
 }
fd9fba
 
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief Action to take when a cluster sync point is hit for a
fd9fba
+ *        PCMK__ATTRD_CMD_UPDATE* message.
fd9fba
+ *
fd9fba
+ * \param[in] xml  The request that should be passed along to
fd9fba
+ *                 attrd_ack_waitlist_clients.  This should be the original
fd9fba
+ *                 IPC request containing the callid for this update message.
fd9fba
+ */
fd9fba
+int
fd9fba
+attrd_cluster_sync_point_update(xmlNode *xml)
fd9fba
+{
fd9fba
+    crm_trace("Hit cluster sync point for attribute update");
fd9fba
+    attrd_ack_waitlist_clients(attrd_sync_point_cluster, xml);
fd9fba
+    return pcmk_rc_ok;
fd9fba
+}
fd9fba
+
fd9fba
 /*!
fd9fba
  * \internal
fd9fba
  * \brief Return the sync point attribute for an IPC request
fd9fba
@@ -268,3 +334,189 @@ attrd_request_has_sync_point(xmlNode *xml)
fd9fba
 {
fd9fba
     return attrd_request_sync_point(xml) != NULL;
fd9fba
 }
fd9fba
+
fd9fba
+static void
fd9fba
+free_action(gpointer data)
fd9fba
+{
fd9fba
+    struct confirmation_action *action = (struct confirmation_action *) data;
fd9fba
+    g_list_free_full(action->respondents, free);
fd9fba
+    free_xml(action->xml);
fd9fba
+    free(action->client_id);
fd9fba
+    free(action);
fd9fba
+}
fd9fba
+
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief When a peer disconnects from the cluster, no longer wait for its confirmation
fd9fba
+ *        for any IPC action.  If this peer is the last one being waited on, this will
fd9fba
+ *        trigger the confirmation action.
fd9fba
+ *
fd9fba
+ * \param[in] host   The disconnecting peer attrd's uname
fd9fba
+ */
fd9fba
+void
fd9fba
+attrd_do_not_expect_from_peer(const char *host)
fd9fba
+{
fd9fba
+    GList *keys = g_hash_table_get_keys(expected_confirmations);
fd9fba
+
fd9fba
+    crm_trace("Removing peer %s from expected confirmations", host);
fd9fba
+
fd9fba
+    for (GList *node = keys; node != NULL; node = node->next) {
fd9fba
+        int callid = *(int *) node->data;
fd9fba
+        attrd_handle_confirmation(callid, host);
fd9fba
+    }
fd9fba
+
fd9fba
+    g_list_free(keys);
fd9fba
+}
fd9fba
+
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief When a client disconnects from the cluster, no longer wait on confirmations
fd9fba
+ *        for it.  Because the peer attrds may still be processing the original IPC
fd9fba
+ *        message, they may still send us confirmations.  However, we will take no
fd9fba
+ *        action on them.
fd9fba
+ *
fd9fba
+ * \param[in] client    The disconnecting client
fd9fba
+ */
fd9fba
+void
fd9fba
+attrd_do_not_wait_for_client(pcmk__client_t *client)
fd9fba
+{
fd9fba
+    GHashTableIter iter;
fd9fba
+    gpointer value;
fd9fba
+
fd9fba
+    if (expected_confirmations == NULL) {
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    g_hash_table_iter_init(&iter, expected_confirmations);
fd9fba
+
fd9fba
+    while (g_hash_table_iter_next(&iter, NULL, &value)) {
fd9fba
+        struct confirmation_action *action = (struct confirmation_action *) value;
fd9fba
+
fd9fba
+        if (pcmk__str_eq(action->client_id, client->id, pcmk__str_none)) {
fd9fba
+            crm_trace("Removing client %s from expected confirmations", client->id);
fd9fba
+            g_hash_table_iter_remove(&iter);
fd9fba
+            crm_trace("%d requests now in expected confirmations table", g_hash_table_size(expected_confirmations));
fd9fba
+            break;
fd9fba
+        }
fd9fba
+    }
fd9fba
+}
fd9fba
+
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief Register some action to be taken when IPC request confirmations are
fd9fba
+ *        received
fd9fba
+ *
fd9fba
+ * When this function is called, a list of all peer attrds that support confirming
fd9fba
+ * requests is generated.  As confirmations from these peer attrds are received,
fd9fba
+ * they are removed from this list.  When the list is empty, the registered action
fd9fba
+ * will be called.
fd9fba
+ *
fd9fba
+ * \note This function should always be called before attrd_send_message is called
fd9fba
+ *       to broadcast to the peers to ensure that we know what replies we are
fd9fba
+ *       waiting on.  Otherwise, it is possible the peer could finish and confirm
fd9fba
+ *       before we know to expect it.
fd9fba
+ *
fd9fba
+ * \param[in] request The request that is awaiting confirmations
fd9fba
+ * \param[in] fn      A function to be run after all confirmations are received
fd9fba
+ */
fd9fba
+void
fd9fba
+attrd_expect_confirmations(pcmk__request_t *request, attrd_confirmation_action_fn fn)
fd9fba
+{
fd9fba
+    struct confirmation_action *action = NULL;
fd9fba
+    GHashTableIter iter;
fd9fba
+    gpointer host, ver;
fd9fba
+    GList *respondents = NULL;
fd9fba
+    int callid;
fd9fba
+
fd9fba
+    if (expected_confirmations == NULL) {
fd9fba
+        expected_confirmations = pcmk__intkey_table((GDestroyNotify) free_action);
fd9fba
+    }
fd9fba
+
fd9fba
+    if (crm_element_value_int(request->xml, XML_LRM_ATTR_CALLID, &callid) == -1) {
fd9fba
+        crm_err("Could not get callid from xml");
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    if (pcmk__intkey_table_lookup(expected_confirmations, callid)) {
fd9fba
+        crm_err("Already waiting on confirmations for call id %d", callid);
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    g_hash_table_iter_init(&iter, peer_protocol_vers);
fd9fba
+    while (g_hash_table_iter_next(&iter, &host, &ver)) {
fd9fba
+        if (GPOINTER_TO_INT(ver) >= 5) {
fd9fba
+            char *s = strdup((char *) host);
fd9fba
+
fd9fba
+            CRM_ASSERT(s != NULL);
fd9fba
+            respondents = g_list_prepend(respondents, s);
fd9fba
+        }
fd9fba
+    }
fd9fba
+
fd9fba
+    action = calloc(1, sizeof(struct confirmation_action));
fd9fba
+    CRM_ASSERT(action != NULL);
fd9fba
+
fd9fba
+    action->respondents = respondents;
fd9fba
+    action->fn = fn;
fd9fba
+    action->xml = copy_xml(request->xml);
fd9fba
+
fd9fba
+    action->client_id = strdup(request->ipc_client->id);
fd9fba
+    CRM_ASSERT(action->client_id != NULL);
fd9fba
+
fd9fba
+    action->ipc_id = request->ipc_id;
fd9fba
+    action->flags = request->flags;
fd9fba
+
fd9fba
+    pcmk__intkey_table_insert(expected_confirmations, callid, action);
fd9fba
+    crm_trace("Callid %d now waiting on %d confirmations", callid, g_list_length(respondents));
fd9fba
+    crm_trace("%d requests now in expected confirmations table", g_hash_table_size(expected_confirmations));
fd9fba
+}
fd9fba
+
fd9fba
+void
fd9fba
+attrd_free_confirmations(void)
fd9fba
+{
fd9fba
+    if (expected_confirmations != NULL) {
fd9fba
+        g_hash_table_destroy(expected_confirmations);
fd9fba
+        expected_confirmations = NULL;
fd9fba
+    }
fd9fba
+}
fd9fba
+
fd9fba
+/*!
fd9fba
+ * \internal
fd9fba
+ * \brief Process a confirmation message from a peer attrd
fd9fba
+ *
fd9fba
+ * This function is called every time a PCMK__ATTRD_CMD_CONFIRM message is
fd9fba
+ * received from a peer attrd.  If this is the last confirmation we are waiting
fd9fba
+ * on for a given operation, the registered action will be called.
fd9fba
+ *
fd9fba
+ * \param[in] callid The unique callid for the XML IPC request
fd9fba
+ * \param[in] host   The confirming peer attrd's uname
fd9fba
+ */
fd9fba
+void
fd9fba
+attrd_handle_confirmation(int callid, const char *host)
fd9fba
+{
fd9fba
+    struct confirmation_action *action = NULL;
fd9fba
+    GList *node = NULL;
fd9fba
+
fd9fba
+    if (expected_confirmations == NULL) {
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    action = pcmk__intkey_table_lookup(expected_confirmations, callid);
fd9fba
+    if (action == NULL) {
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    node = g_list_find_custom(action->respondents, host, (GCompareFunc) strcasecmp);
fd9fba
+
fd9fba
+    if (node == NULL) {
fd9fba
+        return;
fd9fba
+    }
fd9fba
+
fd9fba
+    action->respondents = g_list_remove(action->respondents, node->data);
fd9fba
+    crm_trace("Callid %d now waiting on %d confirmations", callid, g_list_length(action->respondents));
fd9fba
+
fd9fba
+    if (action->respondents == NULL) {
fd9fba
+        action->fn(action->xml);
fd9fba
+        pcmk__intkey_table_remove(expected_confirmations, callid);
fd9fba
+        crm_trace("%d requests now in expected confirmations table", g_hash_table_size(expected_confirmations));
fd9fba
+    }
fd9fba
+}
fd9fba
diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c
fd9fba