fcdfbd
From de05f6b52c667155d262ceeb541dc1041d079d71 Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Thu, 8 Sep 2022 11:36:58 -0400
fcdfbd
Subject: [PATCH 01/26] Refactor: tools: Use a uint32_t for attr_options.
fcdfbd
fcdfbd
---
fcdfbd
 tools/attrd_updater.c | 2 +-
fcdfbd
 1 file changed, 1 insertion(+), 1 deletion(-)
fcdfbd
fcdfbd
diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c
fcdfbd
index d90567a..b85a281 100644
fcdfbd
--- a/tools/attrd_updater.c
fcdfbd
+++ b/tools/attrd_updater.c
fcdfbd
@@ -47,7 +47,7 @@ struct {
fcdfbd
     gchar *attr_node;
fcdfbd
     gchar *attr_set;
fcdfbd
     char *attr_value;
fcdfbd
-    int attr_options;
fcdfbd
+    uint32_t attr_options;
fcdfbd
     gboolean query_all;
fcdfbd
     gboolean quiet;
fcdfbd
 } options = {
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From c6637520b474d44553ade52c0dbe9e36e873135f Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Fri, 21 Oct 2022 14:31:16 -0400
fcdfbd
Subject: [PATCH 02/26] Refactor: libcrmcommon: Make pcmk__xe_match more
fcdfbd
 broadly useful.
fcdfbd
fcdfbd
If attr_v is NULL, simply return the first node with a matching name.
fcdfbd
---
fcdfbd
 lib/common/xml.c | 10 ++++++----
fcdfbd
 1 file changed, 6 insertions(+), 4 deletions(-)
fcdfbd
fcdfbd
diff --git a/lib/common/xml.c b/lib/common/xml.c
fcdfbd
index 036dd87..ac6f46a 100644
fcdfbd
--- a/lib/common/xml.c
fcdfbd
+++ b/lib/common/xml.c
fcdfbd
@@ -510,7 +510,7 @@ find_xml_node(const xmlNode *root, const char *search_path, gboolean must_find)
fcdfbd
  * \param[in] parent     XML element to search
fcdfbd
  * \param[in] node_name  If not NULL, only match children of this type
fcdfbd
  * \param[in] attr_n     If not NULL, only match children with an attribute
fcdfbd
- *                       of this name and a value of \p attr_v
fcdfbd
+ *                       of this name.
fcdfbd
  * \param[in] attr_v     If \p attr_n and this are not NULL, only match children
fcdfbd
  *                       with an attribute named \p attr_n and this value
fcdfbd
  *
fcdfbd
@@ -520,14 +520,16 @@ xmlNode *
fcdfbd
 pcmk__xe_match(const xmlNode *parent, const char *node_name,
fcdfbd
                const char *attr_n, const char *attr_v)
fcdfbd
 {
fcdfbd
-    /* ensure attr_v specified when attr_n is */
fcdfbd
-    CRM_CHECK(attr_n == NULL || attr_v != NULL, return NULL);
fcdfbd
+    CRM_CHECK(parent != NULL, return NULL);
fcdfbd
+    CRM_CHECK(attr_v == NULL || attr_n != NULL, return NULL);
fcdfbd
 
fcdfbd
     for (xmlNode *child = pcmk__xml_first_child(parent); child != NULL;
fcdfbd
          child = pcmk__xml_next(child)) {
fcdfbd
         if (pcmk__str_eq(node_name, (const char *) (child->name),
fcdfbd
                          pcmk__str_null_matches)
fcdfbd
-            && ((attr_n == NULL) || attr_matches(child, attr_n, attr_v))) {
fcdfbd
+            && ((attr_n == NULL) ||
fcdfbd
+                (attr_v == NULL && xmlHasProp(child, (pcmkXmlStr) attr_n)) ||
fcdfbd
+                (attr_v != NULL && attr_matches(child, attr_n, attr_v)))) {
fcdfbd
             return child;
fcdfbd
         }
fcdfbd
     }
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From dd520579484c6ec091f7fbb550347941302dad0e Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Fri, 21 Oct 2022 14:32:46 -0400
fcdfbd
Subject: [PATCH 03/26] Tests: libcrmcommon: Add tests for pcmk__xe_match.
fcdfbd
fcdfbd
---
fcdfbd
 lib/common/tests/xml/Makefile.am           |   3 +-
fcdfbd
 lib/common/tests/xml/pcmk__xe_match_test.c | 105 +++++++++++++++++++++
fcdfbd
 2 files changed, 107 insertions(+), 1 deletion(-)
fcdfbd
 create mode 100644 lib/common/tests/xml/pcmk__xe_match_test.c
fcdfbd
fcdfbd
diff --git a/lib/common/tests/xml/Makefile.am b/lib/common/tests/xml/Makefile.am
fcdfbd
index 342ca07..0ccdcc3 100644
fcdfbd
--- a/lib/common/tests/xml/Makefile.am
fcdfbd
+++ b/lib/common/tests/xml/Makefile.am
fcdfbd
@@ -11,6 +11,7 @@ include $(top_srcdir)/mk/tap.mk
fcdfbd
 include $(top_srcdir)/mk/unittest.mk
fcdfbd
 
fcdfbd
 # Add "_test" to the end of all test program names to simplify .gitignore.
fcdfbd
-check_PROGRAMS =	pcmk__xe_foreach_child_test
fcdfbd
+check_PROGRAMS =	pcmk__xe_foreach_child_test \
fcdfbd
+					pcmk__xe_match_test
fcdfbd
 
fcdfbd
 TESTS = $(check_PROGRAMS)
fcdfbd
diff --git a/lib/common/tests/xml/pcmk__xe_match_test.c b/lib/common/tests/xml/pcmk__xe_match_test.c
fcdfbd
new file mode 100644
fcdfbd
index 0000000..fd529ba
fcdfbd
--- /dev/null
fcdfbd
+++ b/lib/common/tests/xml/pcmk__xe_match_test.c
fcdfbd
@@ -0,0 +1,105 @@
fcdfbd
+/*
fcdfbd
+ * Copyright 2022 the Pacemaker project contributors
fcdfbd
+ *
fcdfbd
+ * The version control history for this file may have further details.
fcdfbd
+ *
fcdfbd
+ * This source code is licensed under the GNU Lesser General Public License
fcdfbd
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
fcdfbd
+ */
fcdfbd
+
fcdfbd
+#include <crm_internal.h>
fcdfbd
+
fcdfbd
+#include <crm/common/unittest_internal.h>
fcdfbd
+#include <crm/common/xml_internal.h>
fcdfbd
+
fcdfbd
+const char *str1 =
fcdfbd
+    "<xml>\n"
fcdfbd
+    "  \n"
fcdfbd
+    "  <nodeA attrA=\"123\" id=\"1\">\n"
fcdfbd
+    "    content\n"
fcdfbd
+    "  </nodeA>\n"
fcdfbd
+    "  \n"
fcdfbd
+    "  <nodeA attrA=\"456\" id=\"2\">\n"
fcdfbd
+    "    content\n"
fcdfbd
+    "  </nodeA>\n"
fcdfbd
+    "  \n"
fcdfbd
+    "  <nodeA attrB=\"XYZ\" id=\"3\">\n"
fcdfbd
+    "    content\n"
fcdfbd
+    "  </nodeA>\n"
fcdfbd
+    "  \n"
fcdfbd
+    "  <nodeB attrA=\"123\" id=\"4\">\n"
fcdfbd
+    "    content\n"
fcdfbd
+    "  </nodeA>\n"
fcdfbd
+    "  \n"
fcdfbd
+    "  <nodeB attrB=\"ABC\" id=\"5\">\n"
fcdfbd
+    "    content\n"
fcdfbd
+    "  </nodeA>\n"
fcdfbd
+    "</xml>";
fcdfbd
+
fcdfbd
+static void
fcdfbd
+bad_input(void **state) {
fcdfbd
+    xmlNode *xml = string2xml(str1);
fcdfbd
+
fcdfbd
+    assert_null(pcmk__xe_match(NULL, NULL, NULL, NULL));
fcdfbd
+    assert_null(pcmk__xe_match(NULL, NULL, NULL, "attrX"));
fcdfbd
+
fcdfbd
+    free_xml(xml);
fcdfbd
+}
fcdfbd
+
fcdfbd
+static void
fcdfbd
+not_found(void **state) {
fcdfbd
+    xmlNode *xml = string2xml(str1);
fcdfbd
+
fcdfbd
+    /* No node with an attrX attribute */
fcdfbd
+    assert_null(pcmk__xe_match(xml, NULL, "attrX", NULL));
fcdfbd
+    /* No nodeX node */
fcdfbd
+    assert_null(pcmk__xe_match(xml, "nodeX", NULL, NULL));
fcdfbd
+    /* No nodeA node with attrX */
fcdfbd
+    assert_null(pcmk__xe_match(xml, "nodeA", "attrX", NULL));
fcdfbd
+    /* No nodeA node with attrA=XYZ */
fcdfbd
+    assert_null(pcmk__xe_match(xml, "nodeA", "attrA", "XYZ"));
fcdfbd
+
fcdfbd
+    free_xml(xml);
fcdfbd
+}
fcdfbd
+
fcdfbd
+static void
fcdfbd
+find_attrB(void **state) {
fcdfbd
+    xmlNode *xml = string2xml(str1);
fcdfbd
+    xmlNode *result = NULL;
fcdfbd
+
fcdfbd
+    /* Find the first node with attrB */
fcdfbd
+    result = pcmk__xe_match(xml, NULL, "attrB", NULL);
fcdfbd
+    assert_non_null(result);
fcdfbd
+    assert_string_equal(crm_element_value(result, "id"), "3");
fcdfbd
+
fcdfbd
+    /* Find the first nodeB with attrB */
fcdfbd
+    result = pcmk__xe_match(xml, "nodeB", "attrB", NULL);
fcdfbd
+    assert_non_null(result);
fcdfbd
+    assert_string_equal(crm_element_value(result, "id"), "5");
fcdfbd
+
fcdfbd
+    free_xml(xml);
fcdfbd
+}
fcdfbd
+
fcdfbd
+static void
fcdfbd
+find_attrA_matching(void **state) {
fcdfbd
+    xmlNode *xml = string2xml(str1);
fcdfbd
+    xmlNode *result = NULL;
fcdfbd
+
fcdfbd
+    /* Find attrA=456 */
fcdfbd
+    result = pcmk__xe_match(xml, NULL, "attrA", "456");
fcdfbd
+    assert_non_null(result);
fcdfbd
+    assert_string_equal(crm_element_value(result, "id"), "2");
fcdfbd
+
fcdfbd
+    /* Find a nodeB with attrA=123 */
fcdfbd
+    result = pcmk__xe_match(xml, "nodeB", "attrA", "123");
fcdfbd
+    assert_non_null(result);
fcdfbd
+    assert_string_equal(crm_element_value(result, "id"), "4");
fcdfbd
+
fcdfbd
+    free_xml(xml);
fcdfbd
+}
fcdfbd
+
fcdfbd
+PCMK__UNIT_TEST(NULL, NULL,
fcdfbd
+                cmocka_unit_test(bad_input),
fcdfbd
+                cmocka_unit_test(not_found),
fcdfbd
+                cmocka_unit_test(find_attrB),
fcdfbd
+                cmocka_unit_test(find_attrA_matching));
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 03af8498d8aaf21c509cec9b0ec4b78475da41d7 Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Thu, 8 Sep 2022 12:22:26 -0400
fcdfbd
Subject: [PATCH 04/26] Feature: libcrmcommon: Add attrd options for specifying
fcdfbd
 a sync point.
fcdfbd
fcdfbd
---
fcdfbd
 include/crm/common/attrd_internal.h | 16 +++++++++-------
fcdfbd
 1 file changed, 9 insertions(+), 7 deletions(-)
fcdfbd
fcdfbd
diff --git a/include/crm/common/attrd_internal.h b/include/crm/common/attrd_internal.h
fcdfbd
index f7033ad..389be48 100644
fcdfbd
--- a/include/crm/common/attrd_internal.h
fcdfbd
+++ b/include/crm/common/attrd_internal.h
fcdfbd
@@ -16,13 +16,15 @@ extern "C" {
fcdfbd
 
fcdfbd
 // Options for clients to use with functions below
fcdfbd
 enum pcmk__node_attr_opts {
fcdfbd
-    pcmk__node_attr_none    = 0,
fcdfbd
-    pcmk__node_attr_remote  = (1 << 0),
fcdfbd
-    pcmk__node_attr_private = (1 << 1),
fcdfbd
-    pcmk__node_attr_pattern = (1 << 2),
fcdfbd
-    pcmk__node_attr_value   = (1 << 3),
fcdfbd
-    pcmk__node_attr_delay   = (1 << 4),
fcdfbd
-    pcmk__node_attr_perm    = (1 << 5),
fcdfbd
+    pcmk__node_attr_none           = 0,
fcdfbd
+    pcmk__node_attr_remote         = (1 << 0),
fcdfbd
+    pcmk__node_attr_private        = (1 << 1),
fcdfbd
+    pcmk__node_attr_pattern        = (1 << 2),
fcdfbd
+    pcmk__node_attr_value          = (1 << 3),
fcdfbd
+    pcmk__node_attr_delay          = (1 << 4),
fcdfbd
+    pcmk__node_attr_perm           = (1 << 5),
fcdfbd
+    pcmk__node_attr_sync_local     = (1 << 6),
fcdfbd
+    pcmk__node_attr_sync_cluster   = (1 << 7),
fcdfbd
 };
fcdfbd
 
fcdfbd
 #define pcmk__set_node_attr_flags(node_attr_flags, flags_to_set) do {   \
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 5c8825293ee21d3823bdcd01b0df9c7d39739940 Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Thu, 8 Sep 2022 12:23:09 -0400
fcdfbd
Subject: [PATCH 05/26] Feature: libcrmcommon: Add sync point to IPC request
fcdfbd
 XML.
fcdfbd
fcdfbd
If one of the pcmk__node_attr_sync_* options is provided, add an
fcdfbd
attribute to the request XML.  This will later be inspected by the
fcdfbd
server to determine when to send the reply to the client.
fcdfbd
---
fcdfbd
 include/crm/common/options_internal.h | 2 ++
fcdfbd
 include/crm_internal.h                | 1 +
fcdfbd
 lib/common/ipc_attrd.c                | 6 ++++++
fcdfbd
 3 files changed, 9 insertions(+)
fcdfbd
fcdfbd
diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h
fcdfbd
index b153c67..f29ba3f 100644
fcdfbd
--- a/include/crm/common/options_internal.h
fcdfbd
+++ b/include/crm/common/options_internal.h
fcdfbd
@@ -145,9 +145,11 @@ bool pcmk__valid_sbd_timeout(const char *value);
fcdfbd
 #define PCMK__META_ALLOW_UNHEALTHY_NODES    "allow-unhealthy-nodes"
fcdfbd
 
fcdfbd
 // Constants for enumerated values for various options
fcdfbd
+#define PCMK__VALUE_CLUSTER                 "cluster"
fcdfbd
 #define PCMK__VALUE_CUSTOM                  "custom"
fcdfbd
 #define PCMK__VALUE_FENCING                 "fencing"
fcdfbd
 #define PCMK__VALUE_GREEN                   "green"
fcdfbd
+#define PCMK__VALUE_LOCAL                   "local"
fcdfbd
 #define PCMK__VALUE_MIGRATE_ON_RED          "migrate-on-red"
fcdfbd
 #define PCMK__VALUE_NONE                    "none"
fcdfbd
 #define PCMK__VALUE_NOTHING                 "nothing"
fcdfbd
diff --git a/include/crm_internal.h b/include/crm_internal.h
fcdfbd
index e6e2e96..08193c3 100644
fcdfbd
--- a/include/crm_internal.h
fcdfbd
+++ b/include/crm_internal.h
fcdfbd
@@ -71,6 +71,7 @@
fcdfbd
 #define PCMK__XA_ATTR_RESOURCE          "attr_resource"
fcdfbd
 #define PCMK__XA_ATTR_SECTION           "attr_section"
fcdfbd
 #define PCMK__XA_ATTR_SET               "attr_set"
fcdfbd
+#define PCMK__XA_ATTR_SYNC_POINT        "attr_sync_point"
fcdfbd
 #define PCMK__XA_ATTR_USER              "attr_user"
fcdfbd
 #define PCMK__XA_ATTR_UUID              "attr_key"
fcdfbd
 #define PCMK__XA_ATTR_VALUE             "attr_value"
fcdfbd
diff --git a/lib/common/ipc_attrd.c b/lib/common/ipc_attrd.c
fcdfbd
index f6cfbc4..4606509 100644
fcdfbd
--- a/lib/common/ipc_attrd.c
fcdfbd
+++ b/lib/common/ipc_attrd.c
fcdfbd
@@ -431,6 +431,12 @@ populate_update_op(xmlNode *op, const char *node, const char *name, const char *
fcdfbd
                     pcmk_is_set(options, pcmk__node_attr_remote));
fcdfbd
     crm_xml_add_int(op, PCMK__XA_ATTR_IS_PRIVATE,
fcdfbd
                     pcmk_is_set(options, pcmk__node_attr_private));
fcdfbd
+
fcdfbd
+    if (pcmk_is_set(options, pcmk__node_attr_sync_local)) {
fcdfbd
+        crm_xml_add(op, PCMK__XA_ATTR_SYNC_POINT, PCMK__VALUE_LOCAL);
fcdfbd
+    } else if (pcmk_is_set(options, pcmk__node_attr_sync_cluster)) {
fcdfbd
+        crm_xml_add(op, PCMK__XA_ATTR_SYNC_POINT, PCMK__VALUE_CLUSTER);
fcdfbd
+    }
fcdfbd
 }
fcdfbd
 
fcdfbd
 int
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From e2b3fee630caf0846ca8bbffcef4d6d2acfd32a5 Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Thu, 8 Sep 2022 12:26:28 -0400
fcdfbd
Subject: [PATCH 06/26] Feature: tools: Add --wait= parameter to attrd_updater.
fcdfbd
fcdfbd
This command line option is used to specify the sync point to use.  For
fcdfbd
the moment, it has no effect.
fcdfbd
---
fcdfbd
 tools/attrd_updater.c | 24 ++++++++++++++++++++++++
fcdfbd
 1 file changed, 24 insertions(+)
fcdfbd
fcdfbd
diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c
fcdfbd
index b85a281..c4779a6 100644
fcdfbd
--- a/tools/attrd_updater.c
fcdfbd
+++ b/tools/attrd_updater.c
fcdfbd
@@ -97,6 +97,22 @@ section_cb (const gchar *option_name, const gchar *optarg, gpointer data, GError
fcdfbd
     return TRUE;
fcdfbd
 }
fcdfbd
 
fcdfbd
+static gboolean
fcdfbd
+wait_cb (const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
fcdfbd
+    if (pcmk__str_eq(optarg, "no", pcmk__str_none)) {
fcdfbd
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
fcdfbd
+        return TRUE;
fcdfbd
+    } else if (pcmk__str_eq(optarg, PCMK__VALUE_LOCAL, pcmk__str_none)) {
fcdfbd
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
fcdfbd
+        pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local);
fcdfbd
+        return TRUE;
fcdfbd
+    } else {
fcdfbd
+        g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_USAGE,
fcdfbd
+                    "--wait= must be one of 'no', 'local', 'cluster'");
fcdfbd
+        return FALSE;
fcdfbd
+    }
fcdfbd
+}
fcdfbd
+
fcdfbd
 #define INDENT "                              "
fcdfbd
 
fcdfbd
 static GOptionEntry required_entries[] = {
fcdfbd
@@ -175,6 +191,14 @@ static GOptionEntry addl_entries[] = {
fcdfbd
       "If this creates a new attribute, never write the attribute to CIB",
fcdfbd
       NULL },
fcdfbd
 
fcdfbd
+    { "wait", 'W', 0, G_OPTION_ARG_CALLBACK, wait_cb,
fcdfbd
+      "Wait for some event to occur before returning.  Values are 'no' (wait\n"
fcdfbd
+      INDENT "only for the attribute daemon to acknowledge the request) or\n"
fcdfbd
+      INDENT "'local' (wait until the change has propagated to where a local\n"
fcdfbd
+      INDENT "query will return the request value, or the value set by a\n"
fcdfbd
+      INDENT "later request).  Default is 'no'.",
fcdfbd
+      "UNTIL" },
fcdfbd
+
fcdfbd
     { NULL }
fcdfbd
 };
fcdfbd
 
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 52d51ab41b2f00e72724ab39835b3db86605a96b Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Thu, 20 Oct 2022 14:40:13 -0400
fcdfbd
Subject: [PATCH 07/26] Feature: daemons: Add functions for checking a request
fcdfbd
 for a sync point.
fcdfbd
fcdfbd
---
fcdfbd
 daemons/attrd/Makefile.am       |  1 +
fcdfbd
 daemons/attrd/attrd_sync.c      | 38 +++++++++++++++++++++++++++++++++
fcdfbd
 daemons/attrd/pacemaker-attrd.h |  3 +++
fcdfbd
 3 files changed, 42 insertions(+)
fcdfbd
 create mode 100644 daemons/attrd/attrd_sync.c
fcdfbd
fcdfbd
diff --git a/daemons/attrd/Makefile.am b/daemons/attrd/Makefile.am
fcdfbd
index 1a3d360..6bb81c4 100644
fcdfbd
--- a/daemons/attrd/Makefile.am
fcdfbd
+++ b/daemons/attrd/Makefile.am
fcdfbd
@@ -32,6 +32,7 @@ pacemaker_attrd_SOURCES	= attrd_alerts.c 	\
fcdfbd
 						  attrd_elections.c \
fcdfbd
 						  attrd_ipc.c 		\
fcdfbd
 						  attrd_messages.c 		\
fcdfbd
+						  attrd_sync.c 		\
fcdfbd
 						  attrd_utils.c 	\
fcdfbd
 						  pacemaker-attrd.c
fcdfbd
 
fcdfbd
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fcdfbd
new file mode 100644
fcdfbd
index 0000000..92759d2
fcdfbd
--- /dev/null
fcdfbd
+++ b/daemons/attrd/attrd_sync.c
fcdfbd
@@ -0,0 +1,38 @@
fcdfbd
+/*
fcdfbd
+ * Copyright 2022 the Pacemaker project contributors
fcdfbd
+ *
fcdfbd
+ * The version control history for this file may have further details.
fcdfbd
+ *
fcdfbd
+ * This source code is licensed under the GNU General Public License version 2
fcdfbd
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
fcdfbd
+ */
fcdfbd
+
fcdfbd
+#include <crm_internal.h>
fcdfbd
+
fcdfbd
+#include <crm/msg_xml.h>
fcdfbd
+#include <crm/common/attrd_internal.h>
fcdfbd
+
fcdfbd
+#include "pacemaker-attrd.h"
fcdfbd
+
fcdfbd
+const char *
fcdfbd
+attrd_request_sync_point(xmlNode *xml)
fcdfbd
+{
fcdfbd
+    if (xml_has_children(xml)) {
fcdfbd
+        xmlNode *child = pcmk__xe_match(xml, XML_ATTR_OP, PCMK__XA_ATTR_SYNC_POINT, NULL);
fcdfbd
+
fcdfbd
+        if (child) {
fcdfbd
+            return crm_element_value(child, PCMK__XA_ATTR_SYNC_POINT);
fcdfbd
+        } else {
fcdfbd
+            return NULL;
fcdfbd
+        }
fcdfbd
+
fcdfbd
+    } else {
fcdfbd
+        return crm_element_value(xml, PCMK__XA_ATTR_SYNC_POINT);
fcdfbd
+    }
fcdfbd
+}
fcdfbd
+
fcdfbd
+bool
fcdfbd
+attrd_request_has_sync_point(xmlNode *xml)
fcdfbd
+{
fcdfbd
+    return attrd_request_sync_point(xml) != NULL;
fcdfbd
+}
fcdfbd
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fcdfbd
index 71ce90a..ff850bb 100644
fcdfbd
--- a/daemons/attrd/pacemaker-attrd.h
fcdfbd
+++ b/daemons/attrd/pacemaker-attrd.h
fcdfbd
@@ -182,4 +182,7 @@ mainloop_timer_t *attrd_add_timer(const char *id, int timeout_ms, attribute_t *a
fcdfbd
 void attrd_unregister_handlers(void);
fcdfbd
 void attrd_handle_request(pcmk__request_t *request);
fcdfbd
 
fcdfbd
+const char *attrd_request_sync_point(xmlNode *xml);
fcdfbd
+bool attrd_request_has_sync_point(xmlNode *xml);
fcdfbd
+
fcdfbd
 #endif /* PACEMAKER_ATTRD__H */
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 2e0509a12ee7d4a612133ee65b75245eea7d271d Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Thu, 20 Oct 2022 14:42:04 -0400
fcdfbd
Subject: [PATCH 08/26] Refactor: daemons: Don't ACK update requests that give
fcdfbd
 a sync point.
fcdfbd
fcdfbd
The ACK is the only response from the server for update messages.  If
fcdfbd
the message specified that it wanted to wait for a sync point, we need
fcdfbd
to delay sending that response until the sync point is reached.
fcdfbd
Therefore, do not always immediately send the ACK.
fcdfbd
---
fcdfbd
 daemons/attrd/attrd_messages.c | 19 ++++++++++++++-----
fcdfbd
 1 file changed, 14 insertions(+), 5 deletions(-)
fcdfbd
fcdfbd
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fcdfbd
index de4a28a..9e8ae40 100644
fcdfbd
--- a/daemons/attrd/attrd_messages.c
fcdfbd
+++ b/daemons/attrd/attrd_messages.c
fcdfbd
@@ -137,12 +137,21 @@ handle_update_request(pcmk__request_t *request)
fcdfbd
         attrd_peer_update(peer, request->xml, host, false);
fcdfbd
         pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fcdfbd
         return NULL;
fcdfbd
+
fcdfbd
     } else {
fcdfbd
-        /* Because attrd_client_update can be called recursively, we send the ACK
fcdfbd
-         * here to ensure that the client only ever receives one.
fcdfbd
-         */
fcdfbd
-        attrd_send_ack(request->ipc_client, request->ipc_id,
fcdfbd
-                       request->flags|crm_ipc_client_response);
fcdfbd
+        if (!attrd_request_has_sync_point(request->xml)) {
fcdfbd
+            /* If the client doesn't want to wait for a sync point, go ahead and send
fcdfbd
+             * the ACK immediately.  Otherwise, we'll send the ACK when the appropriate
fcdfbd
+             * sync point is reached.
fcdfbd
+             *
fcdfbd
+             * In the normal case, attrd_client_update can be called recursively which
fcdfbd
+             * makes where to send the ACK tricky.  Doing it here ensures the client
fcdfbd
+             * only ever receives one.
fcdfbd
+             */
fcdfbd
+            attrd_send_ack(request->ipc_client, request->ipc_id,
fcdfbd
+                           request->flags|crm_ipc_client_response);
fcdfbd
+        }
fcdfbd
+
fcdfbd
         return attrd_client_update(request);
fcdfbd
     }
fcdfbd
 }
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 2a0ff66cdf0085c4c8ab1992ef7e785a4facc8c7 Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Thu, 20 Oct 2022 14:48:48 -0400
fcdfbd
Subject: [PATCH 09/26] Feature: daemons: Add support for local sync points on
fcdfbd
 updates.
fcdfbd
fcdfbd
In the IPC dispatcher for attrd, add the client to a wait list if its
fcdfbd
request specifies a sync point.  When the attribute's value is changed
fcdfbd
on the local attrd, alert any clients waiting on a local sync point by
fcdfbd
then sending the previously delayed ACK.
fcdfbd
fcdfbd
Sync points for other requests and the global sync point are not yet
fcdfbd
supported.
fcdfbd
fcdfbd
Fixes T35.
fcdfbd
---
fcdfbd
 daemons/attrd/attrd_corosync.c  |  18 +++++
fcdfbd
 daemons/attrd/attrd_messages.c  |  12 ++-
fcdfbd
 daemons/attrd/attrd_sync.c      | 137 ++++++++++++++++++++++++++++++++
fcdfbd
 daemons/attrd/pacemaker-attrd.h |   7 ++
fcdfbd
 4 files changed, 173 insertions(+), 1 deletion(-)
fcdfbd
fcdfbd
diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
fcdfbd
index 539e5bf..4337280 100644
fcdfbd
--- a/daemons/attrd/attrd_corosync.c
fcdfbd
+++ b/daemons/attrd/attrd_corosync.c
fcdfbd
@@ -568,14 +568,32 @@ void
fcdfbd
 attrd_peer_update(const crm_node_t *peer, xmlNode *xml, const char *host,
fcdfbd
                   bool filter)
fcdfbd
 {
fcdfbd
+    bool handle_sync_point = false;
fcdfbd
+
fcdfbd
     if (xml_has_children(xml)) {
fcdfbd
         for (xmlNode *child = first_named_child(xml, XML_ATTR_OP); child != NULL;
fcdfbd
              child = crm_next_same_xml(child)) {
fcdfbd
             copy_attrs(xml, child);
fcdfbd
             attrd_peer_update_one(peer, child, filter);
fcdfbd
+
fcdfbd
+            if (attrd_request_has_sync_point(child)) {
fcdfbd
+                handle_sync_point = true;
fcdfbd
+            }
fcdfbd
         }
fcdfbd
 
fcdfbd
     } else {
fcdfbd
         attrd_peer_update_one(peer, xml, filter);
fcdfbd
+
fcdfbd
+        if (attrd_request_has_sync_point(xml)) {
fcdfbd
+            handle_sync_point = true;
fcdfbd
+        }
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    /* If the update XML specified that the client wanted to wait for a sync
fcdfbd
+     * point, process that now.
fcdfbd
+     */
fcdfbd
+    if (handle_sync_point) {
fcdfbd
+        crm_debug("Hit local sync point for attribute update");
fcdfbd
+        attrd_ack_waitlist_clients(attrd_sync_point_local, xml);
fcdfbd
     }
fcdfbd
 }
fcdfbd
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fcdfbd
index 9e8ae40..c96700f 100644
fcdfbd
--- a/daemons/attrd/attrd_messages.c
fcdfbd
+++ b/daemons/attrd/attrd_messages.c
fcdfbd
@@ -139,7 +139,17 @@ handle_update_request(pcmk__request_t *request)
fcdfbd
         return NULL;
fcdfbd
 
fcdfbd
     } else {
fcdfbd
-        if (!attrd_request_has_sync_point(request->xml)) {
fcdfbd
+        if (attrd_request_has_sync_point(request->xml)) {
fcdfbd
+            /* If this client supplied a sync point it wants to wait for, add it to
fcdfbd
+             * the wait list.  Clients on this list will not receive an ACK until
fcdfbd
+             * their sync point is hit which will result in the client stalled there
fcdfbd
+             * until it receives a response.
fcdfbd
+             *
fcdfbd
+             * All other clients will receive the expected response as normal.
fcdfbd
+             */
fcdfbd
+            attrd_add_client_to_waitlist(request);
fcdfbd
+
fcdfbd
+        } else {
fcdfbd
             /* If the client doesn't want to wait for a sync point, go ahead and send
fcdfbd
              * the ACK immediately.  Otherwise, we'll send the ACK when the appropriate
fcdfbd
              * sync point is reached.
fcdfbd
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fcdfbd
index 92759d2..2981bd0 100644
fcdfbd
--- a/daemons/attrd/attrd_sync.c
fcdfbd
+++ b/daemons/attrd/attrd_sync.c
fcdfbd
@@ -14,6 +14,143 @@
fcdfbd
 
fcdfbd
 #include "pacemaker-attrd.h"
fcdfbd
 
fcdfbd
+/* A hash table storing clients that are waiting on a sync point to be reached.
fcdfbd
+ * The key is waitlist_client - just a plain int.  The obvious key would be
fcdfbd
+ * the IPC client's ID, but this is not guaranteed to be unique.  A single client
fcdfbd
+ * could be waiting on a sync point for multiple attributes at the same time.
fcdfbd
+ *
fcdfbd
+ * It is not expected that this hash table will ever be especially large.
fcdfbd
+ */
fcdfbd
+static GHashTable *waitlist = NULL;
fcdfbd
+static int waitlist_client = 0;
fcdfbd
+
fcdfbd
+struct waitlist_node {
fcdfbd
+    /* What kind of sync point does this node describe? */
fcdfbd
+    enum attrd_sync_point sync_point;
fcdfbd
+
fcdfbd
+    /* Information required to construct and send a reply to the client. */
fcdfbd
+    char *client_id;
fcdfbd
+    uint32_t ipc_id;
fcdfbd
+    uint32_t flags;
fcdfbd
+};
fcdfbd
+
fcdfbd
+static void
fcdfbd
+next_key(void)
fcdfbd
+{
fcdfbd
+    do {
fcdfbd
+        waitlist_client++;
fcdfbd
+        if (waitlist_client < 0) {
fcdfbd
+            waitlist_client = 1;
fcdfbd
+        }
fcdfbd
+    } while (g_hash_table_contains(waitlist, GINT_TO_POINTER(waitlist_client)));
fcdfbd
+}
fcdfbd
+
fcdfbd
+static void
fcdfbd
+free_waitlist_node(gpointer data)
fcdfbd
+{
fcdfbd
+    struct waitlist_node *wl = (struct waitlist_node *) data;
fcdfbd
+
fcdfbd
+    free(wl->client_id);
fcdfbd
+    free(wl);
fcdfbd
+}
fcdfbd
+
fcdfbd
+static const char *
fcdfbd
+sync_point_str(enum attrd_sync_point sync_point)
fcdfbd
+{
fcdfbd
+    if (sync_point == attrd_sync_point_local) {
fcdfbd
+        return PCMK__VALUE_LOCAL;
fcdfbd
+    } else if  (sync_point == attrd_sync_point_cluster) {
fcdfbd
+        return PCMK__VALUE_CLUSTER;
fcdfbd
+    } else {
fcdfbd
+        return "unknown";
fcdfbd
+    }
fcdfbd
+}
fcdfbd
+
fcdfbd
+void
fcdfbd
+attrd_add_client_to_waitlist(pcmk__request_t *request)
fcdfbd
+{
fcdfbd
+    const char *sync_point = attrd_request_sync_point(request->xml);
fcdfbd
+    struct waitlist_node *wl = NULL;
fcdfbd
+
fcdfbd
+    if (sync_point == NULL) {
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    if (waitlist == NULL) {
fcdfbd
+        waitlist = pcmk__intkey_table(free_waitlist_node);
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    wl = calloc(sizeof(struct waitlist_node), 1);
fcdfbd
+
fcdfbd
+    CRM_ASSERT(wl != NULL);
fcdfbd
+
fcdfbd
+    wl->client_id = strdup(request->ipc_client->id);
fcdfbd
+
fcdfbd
+    CRM_ASSERT(wl->client_id);
fcdfbd
+
fcdfbd
+    if (pcmk__str_eq(sync_point, PCMK__VALUE_LOCAL, pcmk__str_none)) {
fcdfbd
+        wl->sync_point = attrd_sync_point_local;
fcdfbd
+    } else if (pcmk__str_eq(sync_point, PCMK__VALUE_CLUSTER, pcmk__str_none)) {
fcdfbd
+        wl->sync_point = attrd_sync_point_cluster;
fcdfbd
+    } else {
fcdfbd
+        free_waitlist_node(wl);
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    wl->ipc_id = request->ipc_id;
fcdfbd
+    wl->flags = request->flags;
fcdfbd
+
fcdfbd
+    crm_debug("Added client %s to waitlist for %s sync point",
fcdfbd
+              wl->client_id, sync_point_str(wl->sync_point));
fcdfbd
+
fcdfbd
+    next_key();
fcdfbd
+    pcmk__intkey_table_insert(waitlist, waitlist_client, wl);
fcdfbd
+
fcdfbd
+    /* And then add the key to the request XML so we can uniquely identify
fcdfbd
+     * it when it comes time to issue the ACK.
fcdfbd
+     */
fcdfbd
+    crm_xml_add_int(request->xml, XML_LRM_ATTR_CALLID, waitlist_client);
fcdfbd
+}
fcdfbd
+
fcdfbd
+void
fcdfbd
+attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fcdfbd
+{
fcdfbd
+    int callid;
fcdfbd
+    gpointer value;
fcdfbd
+
fcdfbd
+    if (waitlist == NULL) {
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    if (crm_element_value_int(xml, XML_LRM_ATTR_CALLID, &callid) == -1) {
fcdfbd
+        crm_warn("Could not get callid from request XML");
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    value = pcmk__intkey_table_lookup(waitlist, callid);
fcdfbd
+    if (value != NULL) {
fcdfbd
+        struct waitlist_node *wl = (struct waitlist_node *) value;
fcdfbd
+        pcmk__client_t *client = NULL;
fcdfbd
+
fcdfbd
+        if (wl->sync_point != sync_point) {
fcdfbd
+            return;
fcdfbd
+        }
fcdfbd
+
fcdfbd
+        crm_debug("Alerting client %s for reached %s sync point",
fcdfbd
+                  wl->client_id, sync_point_str(wl->sync_point));
fcdfbd
+
fcdfbd
+        client = pcmk__find_client_by_id(wl->client_id);
fcdfbd
+        if (client == NULL) {
fcdfbd
+            return;
fcdfbd
+        }
fcdfbd
+
fcdfbd
+        attrd_send_ack(client, wl->ipc_id, wl->flags | crm_ipc_client_response);
fcdfbd
+
fcdfbd
+        /* And then remove the client so it doesn't get alerted again. */
fcdfbd
+        pcmk__intkey_table_remove(waitlist, callid);
fcdfbd
+    }
fcdfbd
+}
fcdfbd
+
fcdfbd
 const char *
fcdfbd
 attrd_request_sync_point(xmlNode *xml)
fcdfbd
 {
fcdfbd
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fcdfbd
index ff850bb..9dd8320 100644
fcdfbd
--- a/daemons/attrd/pacemaker-attrd.h
fcdfbd
+++ b/daemons/attrd/pacemaker-attrd.h
fcdfbd
@@ -182,6 +182,13 @@ mainloop_timer_t *attrd_add_timer(const char *id, int timeout_ms, attribute_t *a
fcdfbd
 void attrd_unregister_handlers(void);
fcdfbd
 void attrd_handle_request(pcmk__request_t *request);
fcdfbd
 
fcdfbd
+enum attrd_sync_point {
fcdfbd
+    attrd_sync_point_local,
fcdfbd
+    attrd_sync_point_cluster,
fcdfbd
+};
fcdfbd
+
fcdfbd
+void attrd_add_client_to_waitlist(pcmk__request_t *request);
fcdfbd
+void attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml);
fcdfbd
 const char *attrd_request_sync_point(xmlNode *xml);
fcdfbd
 bool attrd_request_has_sync_point(xmlNode *xml);
fcdfbd
 
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 59caaf1682191a91d6062358b770f8b9457ba3eb Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Thu, 20 Oct 2022 14:56:58 -0400
fcdfbd
Subject: [PATCH 10/26] Feature: daemons: If a client disconnects, remove it
fcdfbd
 from the waitlist.
fcdfbd
fcdfbd
---
fcdfbd
 daemons/attrd/attrd_ipc.c       |  5 +++++
fcdfbd
 daemons/attrd/attrd_sync.c      | 21 +++++++++++++++++++++
fcdfbd
 daemons/attrd/pacemaker-attrd.h |  1 +
fcdfbd
 3 files changed, 27 insertions(+)
fcdfbd
fcdfbd
diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c
fcdfbd
index 7e4a1c0..8aa39c2 100644
fcdfbd
--- a/daemons/attrd/attrd_ipc.c
fcdfbd
+++ b/daemons/attrd/attrd_ipc.c
fcdfbd
@@ -438,8 +438,13 @@ attrd_ipc_closed(qb_ipcs_connection_t *c)
fcdfbd
         crm_trace("Ignoring request to clean up unknown connection %p", c);
fcdfbd
     } else {
fcdfbd
         crm_trace("Cleaning up closed client connection %p", c);
fcdfbd
+
fcdfbd
+        /* Remove the client from the sync point waitlist if it's present. */
fcdfbd
+        attrd_remove_client_from_waitlist(client);
fcdfbd
+
fcdfbd
         pcmk__free_client(client);
fcdfbd
     }
fcdfbd
+
fcdfbd
     return FALSE;
fcdfbd
 }
fcdfbd
 
fcdfbd
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fcdfbd
index 2981bd0..7293318 100644
fcdfbd
--- a/daemons/attrd/attrd_sync.c
fcdfbd
+++ b/daemons/attrd/attrd_sync.c
fcdfbd
@@ -112,6 +112,27 @@ attrd_add_client_to_waitlist(pcmk__request_t *request)
fcdfbd
     crm_xml_add_int(request->xml, XML_LRM_ATTR_CALLID, waitlist_client);
fcdfbd
 }
fcdfbd
 
fcdfbd
+void
fcdfbd
+attrd_remove_client_from_waitlist(pcmk__client_t *client)
fcdfbd
+{
fcdfbd
+    GHashTableIter iter;
fcdfbd
+    gpointer value;
fcdfbd
+
fcdfbd
+    if (waitlist == NULL) {
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    g_hash_table_iter_init(&iter, waitlist);
fcdfbd
+
fcdfbd
+    while (g_hash_table_iter_next(&iter, NULL, &value)) {
fcdfbd
+        struct waitlist_node *wl = (struct waitlist_node *) value;
fcdfbd
+
fcdfbd
+        if (wl->client_id == client->id) {
fcdfbd
+            g_hash_table_iter_remove(&iter);
fcdfbd
+        }
fcdfbd
+    }
fcdfbd
+}
fcdfbd
+
fcdfbd
 void
fcdfbd
 attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fcdfbd
 {
fcdfbd
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fcdfbd
index 9dd8320..b6ecb75 100644
fcdfbd
--- a/daemons/attrd/pacemaker-attrd.h
fcdfbd
+++ b/daemons/attrd/pacemaker-attrd.h
fcdfbd
@@ -189,6 +189,7 @@ enum attrd_sync_point {
fcdfbd
 
fcdfbd
 void attrd_add_client_to_waitlist(pcmk__request_t *request);
fcdfbd
 void attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml);
fcdfbd
+void attrd_remove_client_from_waitlist(pcmk__client_t *client);
fcdfbd
 const char *attrd_request_sync_point(xmlNode *xml);
fcdfbd
 bool attrd_request_has_sync_point(xmlNode *xml);
fcdfbd
 
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From b28042e1d64b48c96dbd9da1e9ee3ff481bbf620 Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Mon, 10 Oct 2022 11:00:20 -0400
fcdfbd
Subject: [PATCH 11/26] Feature: daemons: Add support for local sync points on
fcdfbd
 clearing failures.
fcdfbd
fcdfbd
attrd_clear_client_failure just calls attrd_client_update underneath, so
fcdfbd
that function will handle all the rest of the sync point functionality
fcdfbd
for us.
fcdfbd
---
fcdfbd
 daemons/attrd/attrd_ipc.c      |  2 --
fcdfbd
 daemons/attrd/attrd_messages.c | 19 +++++++++++++++++++
fcdfbd
 2 files changed, 19 insertions(+), 2 deletions(-)
fcdfbd
fcdfbd
diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c
fcdfbd
index 8aa39c2..2e614e8 100644
fcdfbd
--- a/daemons/attrd/attrd_ipc.c
fcdfbd
+++ b/daemons/attrd/attrd_ipc.c
fcdfbd
@@ -101,8 +101,6 @@ attrd_client_clear_failure(pcmk__request_t *request)
fcdfbd
     xmlNode *xml = request->xml;
fcdfbd
     const char *rsc, *op, *interval_spec;
fcdfbd
 
fcdfbd
-    attrd_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags);
fcdfbd
-
fcdfbd
     if (minimum_protocol_version >= 2) {
fcdfbd
         /* Propagate to all peers (including ourselves).
fcdfbd
          * This ends up at attrd_peer_message().
fcdfbd
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fcdfbd
index c96700f..3ba14a6 100644
fcdfbd
--- a/daemons/attrd/attrd_messages.c
fcdfbd
+++ b/daemons/attrd/attrd_messages.c
fcdfbd
@@ -42,6 +42,25 @@ handle_clear_failure_request(pcmk__request_t *request)
fcdfbd
         pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fcdfbd
         return NULL;
fcdfbd
     } else {
fcdfbd
+        if (attrd_request_has_sync_point(request->xml)) {
fcdfbd
+            /* If this client supplied a sync point it wants to wait for, add it to
fcdfbd
+             * the wait list.  Clients on this list will not receive an ACK until
fcdfbd
+             * their sync point is hit which will result in the client stalled there
fcdfbd
+             * until it receives a response.
fcdfbd
+             *
fcdfbd
+             * All other clients will receive the expected response as normal.
fcdfbd
+             */
fcdfbd
+            attrd_add_client_to_waitlist(request);
fcdfbd
+
fcdfbd
+        } else {
fcdfbd
+            /* If the client doesn't want to wait for a sync point, go ahead and send
fcdfbd
+             * the ACK immediately.  Otherwise, we'll send the ACK when the appropriate
fcdfbd
+             * sync point is reached.
fcdfbd
+             */
fcdfbd
+            attrd_send_ack(request->ipc_client, request->ipc_id,
fcdfbd
+                           request->ipc_flags);
fcdfbd
+        }
fcdfbd
+
fcdfbd
         return attrd_client_clear_failure(request);
fcdfbd
     }
fcdfbd
 }
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 291dc3b91e57f2584bbf88cfbe3a360e0332e814 Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Mon, 10 Oct 2022 13:17:24 -0400
fcdfbd
Subject: [PATCH 12/26] Refactor: daemons: Free the waitlist on attrd exit.
fcdfbd
fcdfbd
---
fcdfbd
 daemons/attrd/attrd_sync.c      | 11 +++++++++++
fcdfbd
 daemons/attrd/attrd_utils.c     |  2 ++
fcdfbd
 daemons/attrd/pacemaker-attrd.c |  1 +
fcdfbd
 daemons/attrd/pacemaker-attrd.h |  1 +
fcdfbd
 4 files changed, 15 insertions(+)
fcdfbd
fcdfbd
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fcdfbd
index 7293318..557e49a 100644
fcdfbd
--- a/daemons/attrd/attrd_sync.c
fcdfbd
+++ b/daemons/attrd/attrd_sync.c
fcdfbd
@@ -112,6 +112,17 @@ attrd_add_client_to_waitlist(pcmk__request_t *request)
fcdfbd
     crm_xml_add_int(request->xml, XML_LRM_ATTR_CALLID, waitlist_client);
fcdfbd
 }
fcdfbd
 
fcdfbd
+void
fcdfbd
+attrd_free_waitlist(void)
fcdfbd
+{
fcdfbd
+    if (waitlist == NULL) {
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    g_hash_table_destroy(waitlist);
fcdfbd
+    waitlist = NULL;
fcdfbd
+}
fcdfbd
+
fcdfbd
 void
fcdfbd
 attrd_remove_client_from_waitlist(pcmk__client_t *client)
fcdfbd
 {
fcdfbd
diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c
fcdfbd
index 6a19009..00b879b 100644
fcdfbd
--- a/daemons/attrd/attrd_utils.c
fcdfbd
+++ b/daemons/attrd/attrd_utils.c
fcdfbd
@@ -93,6 +93,8 @@ attrd_shutdown(int nsig)
fcdfbd
     mainloop_destroy_signal(SIGUSR2);
fcdfbd
     mainloop_destroy_signal(SIGTRAP);
fcdfbd
 
fcdfbd
+    attrd_free_waitlist();
fcdfbd
+
fcdfbd
     if ((mloop == NULL) || !g_main_loop_is_running(mloop)) {
fcdfbd
         /* If there's no main loop active, just exit. This should be possible
fcdfbd
          * only if we get SIGTERM in brief windows at start-up and shutdown.
fcdfbd
diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c
fcdfbd
index 2100db4..1336542 100644
fcdfbd
--- a/daemons/attrd/pacemaker-attrd.c
fcdfbd
+++ b/daemons/attrd/pacemaker-attrd.c
fcdfbd
@@ -300,6 +300,7 @@ main(int argc, char **argv)
fcdfbd
         attrd_ipc_fini();
fcdfbd
         attrd_lrmd_disconnect();
fcdfbd
         attrd_cib_disconnect();
fcdfbd
+        attrd_free_waitlist();
fcdfbd
         g_hash_table_destroy(attributes);
fcdfbd
     }
fcdfbd
 
fcdfbd
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fcdfbd
index b6ecb75..537bf85 100644
fcdfbd
--- a/daemons/attrd/pacemaker-attrd.h
fcdfbd
+++ b/daemons/attrd/pacemaker-attrd.h
fcdfbd
@@ -52,6 +52,7 @@ void attrd_run_mainloop(void);
fcdfbd
 
fcdfbd
 void attrd_set_requesting_shutdown(void);
fcdfbd
 void attrd_clear_requesting_shutdown(void);
fcdfbd
+void attrd_free_waitlist(void);
fcdfbd
 bool attrd_requesting_shutdown(void);
fcdfbd
 bool attrd_shutting_down(void);
fcdfbd
 void attrd_shutdown(int nsig);
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 7715ce617c520e14687a82e11ff794c93cd7f64a Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Mon, 10 Oct 2022 13:21:16 -0400
fcdfbd
Subject: [PATCH 13/26] Feature: includes: Bump CRM_FEATURE_SET for local sync
fcdfbd
 points.
fcdfbd
fcdfbd
---
fcdfbd
 include/crm/crm.h | 2 +-
fcdfbd
 1 file changed, 1 insertion(+), 1 deletion(-)
fcdfbd
fcdfbd
diff --git a/include/crm/crm.h b/include/crm/crm.h
fcdfbd
index 5710e4b..7c5c602 100644
fcdfbd
--- a/include/crm/crm.h
fcdfbd
+++ b/include/crm/crm.h
fcdfbd
@@ -66,7 +66,7 @@ extern "C" {
fcdfbd
  * >=3.0.13: Fail counts include operation name and interval
fcdfbd
  * >=3.2.0:  DC supports PCMK_EXEC_INVALID and PCMK_EXEC_NOT_CONNECTED
fcdfbd
  */
fcdfbd
-#  define CRM_FEATURE_SET		"3.16.1"
fcdfbd
+#  define CRM_FEATURE_SET		"3.16.2"
fcdfbd
 
fcdfbd
 /* Pacemaker's CPG protocols use fixed-width binary fields for the sender and
fcdfbd
  * recipient of a CPG message. This imposes an arbitrary limit on cluster node
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From b9054425a76d03f538cd0b3ae27490b1874eee8a Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Fri, 28 Oct 2022 14:23:49 -0400
fcdfbd
Subject: [PATCH 14/26] Refactor: daemons: Add comments for previously added
fcdfbd
 sync point code.
fcdfbd
fcdfbd
---
fcdfbd
 daemons/attrd/attrd_sync.c | 63 ++++++++++++++++++++++++++++++++++++++
fcdfbd
 1 file changed, 63 insertions(+)
fcdfbd
fcdfbd
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fcdfbd
index 557e49a..e9690b5 100644
fcdfbd
--- a/daemons/attrd/attrd_sync.c
fcdfbd
+++ b/daemons/attrd/attrd_sync.c
fcdfbd
@@ -66,6 +66,20 @@ sync_point_str(enum attrd_sync_point sync_point)
fcdfbd
     }
fcdfbd
 }
fcdfbd
 
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief Add a client to the attrd waitlist
fcdfbd
+ *
fcdfbd
+ * Typically, a client receives an ACK for its XML IPC request immediately.  However,
fcdfbd
+ * some clients want to wait until their request has been processed and taken effect.
fcdfbd
+ * This is called a sync point.  Any client placed on this waitlist will have its
fcdfbd
+ * ACK message delayed until either its requested sync point is hit, or until it
fcdfbd
+ * times out.
fcdfbd
+ *
fcdfbd
+ * The XML IPC request must specify the type of sync point it wants to wait for.
fcdfbd
+ *
fcdfbd
+ * \param[in,out] request   The request describing the client to place on the waitlist.
fcdfbd
+ */
fcdfbd
 void
fcdfbd
 attrd_add_client_to_waitlist(pcmk__request_t *request)
fcdfbd
 {
fcdfbd
@@ -112,6 +126,11 @@ attrd_add_client_to_waitlist(pcmk__request_t *request)
fcdfbd
     crm_xml_add_int(request->xml, XML_LRM_ATTR_CALLID, waitlist_client);
fcdfbd
 }
fcdfbd
 
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief Free all memory associated with the waitlist.  This is most typically
fcdfbd
+ *        used when attrd shuts down.
fcdfbd
+ */
fcdfbd
 void
fcdfbd
 attrd_free_waitlist(void)
fcdfbd
 {
fcdfbd
@@ -123,6 +142,13 @@ attrd_free_waitlist(void)
fcdfbd
     waitlist = NULL;
fcdfbd
 }
fcdfbd
 
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief Unconditionally remove a client from the waitlist, such as when the client
fcdfbd
+ *        node disconnects from the cluster
fcdfbd
+ *
fcdfbd
+ * \param[in] client    The client to remove
fcdfbd
+ */
fcdfbd
 void
fcdfbd
 attrd_remove_client_from_waitlist(pcmk__client_t *client)
fcdfbd
 {
fcdfbd
@@ -144,6 +170,18 @@ attrd_remove_client_from_waitlist(pcmk__client_t *client)
fcdfbd
     }
fcdfbd
 }
fcdfbd
 
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief Send an IPC ACK message to all awaiting clients
fcdfbd
+ *
fcdfbd
+ * This function will search the waitlist for all clients that are currently awaiting
fcdfbd
+ * an ACK indicating their attrd operation is complete.  Only those clients with a
fcdfbd
+ * matching sync point type and callid from their original XML IPC request will be
fcdfbd
+ * ACKed.  Once they have received an ACK, they will be removed from the waitlist.
fcdfbd
+ *
fcdfbd
+ * \param[in] sync_point What kind of sync point have we hit?
fcdfbd
+ * \param[in] xml        The original XML IPC request.
fcdfbd
+ */
fcdfbd
 void
fcdfbd
 attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fcdfbd
 {
fcdfbd
@@ -183,6 +221,23 @@ attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fcdfbd
     }
fcdfbd
 }
fcdfbd
 
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief Return the sync point attribute for an IPC request
fcdfbd
+ *
fcdfbd
+ * This function will check both the top-level element of \p xml for a sync
fcdfbd
+ * point attribute, as well as all of its \p op children, if any.  The latter
fcdfbd
+ * is useful for newer versions of attrd that can put multiple IPC requests
fcdfbd
+ * into a single message.
fcdfbd
+ *
fcdfbd
+ * \param[in] xml   An XML IPC request
fcdfbd
+ *
fcdfbd
+ * \note It is assumed that if one child element has a sync point attribute,
fcdfbd
+ *       all will have a sync point attribute and they will all be the same
fcdfbd
+ *       sync point.  No other configuration is supported.
fcdfbd
+ *
fcdfbd
+ * \return The sync point attribute of \p xml, or NULL if none.
fcdfbd
+ */
fcdfbd
 const char *
fcdfbd
 attrd_request_sync_point(xmlNode *xml)
fcdfbd
 {
fcdfbd
@@ -200,6 +255,14 @@ attrd_request_sync_point(xmlNode *xml)
fcdfbd
     }
fcdfbd
 }
fcdfbd
 
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief Does an IPC request contain any sync point attribute?
fcdfbd
+ *
fcdfbd
+ * \param[in] xml   An XML IPC request
fcdfbd
+ *
fcdfbd
+ * \return true if there's a sync point attribute, false otherwise
fcdfbd
+ */
fcdfbd
 bool
fcdfbd
 attrd_request_has_sync_point(xmlNode *xml)
fcdfbd
 {
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 64219fb7075ee58d29f94f077a3b8f94174bb32a Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Wed, 26 Oct 2022 12:43:05 -0400
fcdfbd
Subject: [PATCH 15/26] Feature: tools: Add --wait=cluster option to
fcdfbd
 attrd_updater.
fcdfbd
fcdfbd
---
fcdfbd
 tools/attrd_updater.c | 10 ++++++++--
fcdfbd
 1 file changed, 8 insertions(+), 2 deletions(-)
fcdfbd
fcdfbd
diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c
fcdfbd
index c4779a6..3cd766d 100644
fcdfbd
--- a/tools/attrd_updater.c
fcdfbd
+++ b/tools/attrd_updater.c
fcdfbd
@@ -106,6 +106,10 @@ wait_cb (const gchar *option_name, const gchar *optarg, gpointer data, GError **
fcdfbd
         pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
fcdfbd
         pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local);
fcdfbd
         return TRUE;
fcdfbd
+    } else if (pcmk__str_eq(optarg, PCMK__VALUE_CLUSTER, pcmk__str_none)) {
fcdfbd
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
fcdfbd
+        pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_sync_cluster);
fcdfbd
+        return TRUE;
fcdfbd
     } else {
fcdfbd
         g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_USAGE,
fcdfbd
                     "--wait= must be one of 'no', 'local', 'cluster'");
fcdfbd
@@ -193,10 +197,12 @@ static GOptionEntry addl_entries[] = {
fcdfbd
 
fcdfbd
     { "wait", 'W', 0, G_OPTION_ARG_CALLBACK, wait_cb,
fcdfbd
       "Wait for some event to occur before returning.  Values are 'no' (wait\n"
fcdfbd
-      INDENT "only for the attribute daemon to acknowledge the request) or\n"
fcdfbd
+      INDENT "only for the attribute daemon to acknowledge the request),\n"
fcdfbd
       INDENT "'local' (wait until the change has propagated to where a local\n"
fcdfbd
       INDENT "query will return the request value, or the value set by a\n"
fcdfbd
-      INDENT "later request).  Default is 'no'.",
fcdfbd
+      INDENT "later request), or 'cluster' (wait until the change has propagated\n"
fcdfbd
+      INDENT "to where a query anywhere on the cluster will return the requested\n"
fcdfbd
+      INDENT "value, or the value set by a later request).  Default is 'no'.",
fcdfbd
       "UNTIL" },
fcdfbd
 
fcdfbd
     { NULL }
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 1bc5511fadf6ad670508bd3a2a55129bde16f774 Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Fri, 16 Sep 2022 14:55:06 -0400
fcdfbd
Subject: [PATCH 16/26] Refactor: daemons: Add a confirm= attribute to attrd
fcdfbd
 messages.
fcdfbd
fcdfbd
This allows informing the originator of a message that the message has
fcdfbd
been received and processed.  As yet, there is no mechanism for handling
fcdfbd
and returning the confirmation, only for requesting it.
fcdfbd
---
fcdfbd
 daemons/attrd/attrd_corosync.c  |  6 +++---
fcdfbd
 daemons/attrd/attrd_ipc.c       | 26 +++++++++++++++++++++-----
fcdfbd
 daemons/attrd/attrd_messages.c  | 11 +++++++++--
fcdfbd
 daemons/attrd/pacemaker-attrd.h |  7 ++++---
fcdfbd
 include/crm_internal.h          |  1 +
fcdfbd
 5 files changed, 38 insertions(+), 13 deletions(-)
fcdfbd
fcdfbd
diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
fcdfbd
index 4337280..e86ca07 100644
fcdfbd
--- a/daemons/attrd/attrd_corosync.c
fcdfbd
+++ b/daemons/attrd/attrd_corosync.c
fcdfbd
@@ -124,7 +124,7 @@ broadcast_local_value(const attribute_t *a)
fcdfbd
 
fcdfbd
     crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
fcdfbd
     attrd_add_value_xml(sync, a, v, false);
fcdfbd
-    attrd_send_message(NULL, sync);
fcdfbd
+    attrd_send_message(NULL, sync, false);
fcdfbd
     free_xml(sync);
fcdfbd
     return v;
fcdfbd
 }
fcdfbd
@@ -387,7 +387,7 @@ broadcast_unseen_local_values(void)
fcdfbd
 
fcdfbd
     if (sync != NULL) {
fcdfbd
         crm_debug("Broadcasting local-only values");
fcdfbd
-        attrd_send_message(NULL, sync);
fcdfbd
+        attrd_send_message(NULL, sync, false);
fcdfbd
         free_xml(sync);
fcdfbd
     }
fcdfbd
 }
fcdfbd
@@ -539,7 +539,7 @@ attrd_peer_sync(crm_node_t *peer, xmlNode *xml)
fcdfbd
     }
fcdfbd
 
fcdfbd
     crm_debug("Syncing values to %s", peer?peer->uname:"everyone");
fcdfbd
-    attrd_send_message(peer, sync);
fcdfbd
+    attrd_send_message(peer, sync, false);
fcdfbd
     free_xml(sync);
fcdfbd
 }
fcdfbd
 
fcdfbd
diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c
fcdfbd
index 2e614e8..0fc5e93 100644
fcdfbd
--- a/daemons/attrd/attrd_ipc.c
fcdfbd
+++ b/daemons/attrd/attrd_ipc.c
fcdfbd
@@ -105,7 +105,7 @@ attrd_client_clear_failure(pcmk__request_t *request)
fcdfbd
         /* Propagate to all peers (including ourselves).
fcdfbd
          * This ends up at attrd_peer_message().
fcdfbd
          */
fcdfbd
-        attrd_send_message(NULL, xml);
fcdfbd
+        attrd_send_message(NULL, xml, false);
fcdfbd
         pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fcdfbd
         return NULL;
fcdfbd
     }
fcdfbd
@@ -184,7 +184,7 @@ attrd_client_peer_remove(pcmk__request_t *request)
fcdfbd
     if (host) {
fcdfbd
         crm_info("Client %s is requesting all values for %s be removed",
fcdfbd
                  pcmk__client_name(request->ipc_client), host);
fcdfbd
-        attrd_send_message(NULL, xml); /* ends up at attrd_peer_message() */
fcdfbd
+        attrd_send_message(NULL, xml, false); /* ends up at attrd_peer_message() */
fcdfbd
         free(host_alloc);
fcdfbd
     } else {
fcdfbd
         crm_info("Ignoring request by client %s to remove all peer values without specifying peer",
fcdfbd
@@ -314,7 +314,7 @@ attrd_client_update(pcmk__request_t *request)
fcdfbd
                 }
fcdfbd
             }
fcdfbd
 
fcdfbd
-            attrd_send_message(NULL, xml);
fcdfbd
+            attrd_send_message(NULL, xml, false);
fcdfbd
             pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fcdfbd
 
fcdfbd
         } else {
fcdfbd
@@ -358,7 +358,7 @@ attrd_client_update(pcmk__request_t *request)
fcdfbd
                 if (status == 0) {
fcdfbd
                     crm_trace("Matched %s with %s", attr, regex);
fcdfbd
                     crm_xml_add(xml, PCMK__XA_ATTR_NAME, attr);
fcdfbd
-                    attrd_send_message(NULL, xml);
fcdfbd
+                    attrd_send_message(NULL, xml, false);
fcdfbd
                 }
fcdfbd
             }
fcdfbd
 
fcdfbd
@@ -388,7 +388,23 @@ attrd_client_update(pcmk__request_t *request)
fcdfbd
     crm_debug("Broadcasting %s[%s]=%s%s", attr, crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME),
fcdfbd
               value, (attrd_election_won()? " (writer)" : ""));
fcdfbd
 
fcdfbd
-    attrd_send_message(NULL, xml); /* ends up at attrd_peer_message() */
fcdfbd
+    if (pcmk__str_eq(attrd_request_sync_point(xml), PCMK__VALUE_CLUSTER, pcmk__str_none)) {
fcdfbd
+        /* The client is waiting on the cluster-wide sync point.  In this case,
fcdfbd
+         * the response ACK is not sent until this attrd broadcasts the update
fcdfbd
+         * and receives its own confirmation back from all peers.
fcdfbd
+         */
fcdfbd
+        attrd_send_message(NULL, xml, true); /* ends up at attrd_peer_message() */
fcdfbd
+
fcdfbd
+    } else {
fcdfbd
+        /* The client is either waiting on the local sync point or was not
fcdfbd
+         * waiting on any sync point at all.  For the local sync point, the
fcdfbd
+         * response ACK is sent in attrd_peer_update.  For clients not
fcdfbd
+         * waiting on any sync point, the response ACK is sent in
fcdfbd
+         * handle_update_request immediately before this function was called.
fcdfbd
+         */
fcdfbd
+        attrd_send_message(NULL, xml, false); /* ends up at attrd_peer_message() */
fcdfbd
+    }
fcdfbd
+
fcdfbd
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fcdfbd
     return NULL;
fcdfbd
 }
fcdfbd
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fcdfbd
index 3ba14a6..78df0d0 100644
fcdfbd
--- a/daemons/attrd/attrd_messages.c
fcdfbd
+++ b/daemons/attrd/attrd_messages.c
fcdfbd
@@ -279,16 +279,23 @@ attrd_broadcast_protocol(void)
fcdfbd
     crm_debug("Broadcasting attrd protocol version %s for node %s",
fcdfbd
               ATTRD_PROTOCOL_VERSION, attrd_cluster->uname);
fcdfbd
 
fcdfbd
-    attrd_send_message(NULL, attrd_op); /* ends up at attrd_peer_message() */
fcdfbd
+    attrd_send_message(NULL, attrd_op, false); /* ends up at attrd_peer_message() */
fcdfbd
 
fcdfbd
     free_xml(attrd_op);
fcdfbd
 }
fcdfbd
 
fcdfbd
 gboolean
fcdfbd
-attrd_send_message(crm_node_t * node, xmlNode * data)
fcdfbd
+attrd_send_message(crm_node_t *node, xmlNode *data, bool confirm)
fcdfbd
 {
fcdfbd
     crm_xml_add(data, F_TYPE, T_ATTRD);
fcdfbd
     crm_xml_add(data, PCMK__XA_ATTR_VERSION, ATTRD_PROTOCOL_VERSION);
fcdfbd
+
fcdfbd
+    /* Request a confirmation from the destination peer node (which could
fcdfbd
+     * be all if node is NULL) that the message has been received and
fcdfbd
+     * acted upon.
fcdfbd
+     */
fcdfbd
+    pcmk__xe_set_bool_attr(data, PCMK__XA_CONFIRM, confirm);
fcdfbd
+
fcdfbd
     attrd_xml_add_writer(data);
fcdfbd
     return send_cluster_message(node, crm_msg_attrd, data, TRUE);
fcdfbd
 }
fcdfbd
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fcdfbd
index 537bf85..25f7c8a 100644
fcdfbd
--- a/daemons/attrd/pacemaker-attrd.h
fcdfbd
+++ b/daemons/attrd/pacemaker-attrd.h
fcdfbd
@@ -39,10 +39,11 @@
fcdfbd
  *                      PCMK__ATTRD_CMD_UPDATE_DELAY
fcdfbd
  *     2       1.1.17   PCMK__ATTRD_CMD_CLEAR_FAILURE
fcdfbd
  *     3       2.1.1    PCMK__ATTRD_CMD_SYNC_RESPONSE indicates remote nodes
fcdfbd
- *     4       2.2.0    Multiple attributes can be updated in a single IPC
fcdfbd
+ *     4       2.1.5    Multiple attributes can be updated in a single IPC
fcdfbd
  *                      message
fcdfbd
+ *     5       2.1.5    Peers can request confirmation of a sent message
fcdfbd
  */
fcdfbd
-#define ATTRD_PROTOCOL_VERSION "4"
fcdfbd
+#define ATTRD_PROTOCOL_VERSION "5"
fcdfbd
 
fcdfbd
 #define attrd_send_ack(client, id, flags) \
fcdfbd
     pcmk__ipc_send_ack((client), (id), (flags), "ack", ATTRD_PROTOCOL_VERSION, CRM_EX_INDETERMINATE)
fcdfbd
@@ -162,7 +163,7 @@ xmlNode *attrd_client_clear_failure(pcmk__request_t *request);
fcdfbd
 xmlNode *attrd_client_update(pcmk__request_t *request);
fcdfbd
 xmlNode *attrd_client_refresh(pcmk__request_t *request);
fcdfbd
 xmlNode *attrd_client_query(pcmk__request_t *request);
fcdfbd
-gboolean attrd_send_message(crm_node_t * node, xmlNode * data);
fcdfbd
+gboolean attrd_send_message(crm_node_t *node, xmlNode *data, bool confirm);
fcdfbd
 
fcdfbd
 xmlNode *attrd_add_value_xml(xmlNode *parent, const attribute_t *a,
fcdfbd
                              const attribute_value_t *v, bool force_write);
fcdfbd
diff --git a/include/crm_internal.h b/include/crm_internal.h
fcdfbd
index 08193c3..63a1726 100644
fcdfbd
--- a/include/crm_internal.h
fcdfbd
+++ b/include/crm_internal.h
fcdfbd
@@ -79,6 +79,7 @@
fcdfbd
 #define PCMK__XA_ATTR_WRITER            "attr_writer"
fcdfbd
 #define PCMK__XA_CONFIG_ERRORS          "config-errors"
fcdfbd
 #define PCMK__XA_CONFIG_WARNINGS        "config-warnings"
fcdfbd
+#define PCMK__XA_CONFIRM                "confirm"
fcdfbd
 #define PCMK__XA_GRAPH_ERRORS           "graph-errors"
fcdfbd
 #define PCMK__XA_GRAPH_WARNINGS         "graph-warnings"
fcdfbd
 #define PCMK__XA_MODE                   "mode"
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 6f389038fc0b11f6291c022c99f188666c65f530 Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Wed, 26 Oct 2022 14:44:42 -0400
fcdfbd
Subject: [PATCH 17/26] Feature: daemons: Respond to received attrd
fcdfbd
 confirmation requests.
fcdfbd
fcdfbd
On the receiving peer side, if the XML request contains confirm="true",
fcdfbd
construct a confirmation message after handling the request completes
fcdfbd
and send it back to the originating peer.
fcdfbd
fcdfbd
On the originating peer side, add a skeleton handler for confirmation
fcdfbd
messages.  This does nothing at the moment except log it.
fcdfbd
---
fcdfbd
 daemons/attrd/attrd_corosync.c | 38 ++++++++++++++++++++++++++++++++++
fcdfbd
 daemons/attrd/attrd_messages.c | 13 ++++++++++++
fcdfbd
 include/crm_internal.h         |  1 +
fcdfbd
 3 files changed, 52 insertions(+)
fcdfbd
fcdfbd
diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
fcdfbd
index e86ca07..1245d9c 100644
fcdfbd
--- a/daemons/attrd/attrd_corosync.c
fcdfbd
+++ b/daemons/attrd/attrd_corosync.c
fcdfbd
@@ -25,6 +25,19 @@
fcdfbd
 
fcdfbd
 extern crm_exit_t attrd_exit_status;
fcdfbd
 
fcdfbd
+static xmlNode *
fcdfbd
+attrd_confirmation(int callid)
fcdfbd
+{
fcdfbd
+    xmlNode *node = create_xml_node(NULL, __func__);
fcdfbd
+
fcdfbd
+    crm_xml_add(node, F_TYPE, T_ATTRD);
fcdfbd
+    crm_xml_add(node, F_ORIG, get_local_node_name());
fcdfbd
+    crm_xml_add(node, PCMK__XA_TASK, PCMK__ATTRD_CMD_CONFIRM);
fcdfbd
+    crm_xml_add_int(node, XML_LRM_ATTR_CALLID, callid);
fcdfbd
+
fcdfbd
+    return node;
fcdfbd
+}
fcdfbd
+
fcdfbd
 static void
fcdfbd
 attrd_peer_message(crm_node_t *peer, xmlNode *xml)
fcdfbd
 {
fcdfbd
@@ -57,6 +70,31 @@ attrd_peer_message(crm_node_t *peer, xmlNode *xml)
fcdfbd
         CRM_CHECK(request.op != NULL, return);
fcdfbd
 
fcdfbd
         attrd_handle_request(&request);
fcdfbd
+
fcdfbd
+        /* Having finished handling the request, check to see if the originating
fcdfbd
+         * peer requested confirmation.  If so, send that confirmation back now.
fcdfbd
+         */
fcdfbd
+        if (pcmk__xe_attr_is_true(xml, PCMK__XA_CONFIRM)) {
fcdfbd
+            int callid = 0;
fcdfbd
+            xmlNode *reply = NULL;
fcdfbd
+
fcdfbd
+            /* Add the confirmation ID for the message we are confirming to the
fcdfbd
+             * response so the originating peer knows what they're a confirmation
fcdfbd
+             * for.
fcdfbd
+             */
fcdfbd
+            crm_element_value_int(xml, XML_LRM_ATTR_CALLID, &callid);
fcdfbd
+            reply = attrd_confirmation(callid);
fcdfbd
+
fcdfbd
+            /* And then send the confirmation back to the originating peer.  This
fcdfbd
+             * ends up right back in this same function (attrd_peer_message) on the
fcdfbd
+             * peer where it will have to do something with a PCMK__XA_CONFIRM type
fcdfbd
+             * message.
fcdfbd
+             */
fcdfbd
+            crm_debug("Sending %s a confirmation", peer->uname);
fcdfbd
+            attrd_send_message(peer, reply, false);
fcdfbd
+            free_xml(reply);
fcdfbd
+        }
fcdfbd
+
fcdfbd
         pcmk__reset_request(&request);
fcdfbd
     }
fcdfbd
 }
fcdfbd
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fcdfbd
index 78df0d0..9c792b2 100644
fcdfbd
--- a/daemons/attrd/attrd_messages.c
fcdfbd
+++ b/daemons/attrd/attrd_messages.c
fcdfbd
@@ -65,6 +65,18 @@ handle_clear_failure_request(pcmk__request_t *request)
fcdfbd
     }
fcdfbd
 }
fcdfbd
 
fcdfbd
+static xmlNode *
fcdfbd
+handle_confirm_request(pcmk__request_t *request)
fcdfbd
+{
fcdfbd
+    if (request->peer != NULL) {
fcdfbd
+        crm_debug("Received confirmation from %s", request->peer);
fcdfbd
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fcdfbd
+        return NULL;
fcdfbd
+    } else {
fcdfbd
+        return handle_unknown_request(request);
fcdfbd
+    }
fcdfbd
+}
fcdfbd
+
fcdfbd
 static xmlNode *
fcdfbd
 handle_flush_request(pcmk__request_t *request)
fcdfbd
 {
fcdfbd
@@ -190,6 +202,7 @@ attrd_register_handlers(void)
fcdfbd
 {
fcdfbd
     pcmk__server_command_t handlers[] = {
fcdfbd
         { PCMK__ATTRD_CMD_CLEAR_FAILURE, handle_clear_failure_request },
fcdfbd
+        { PCMK__ATTRD_CMD_CONFIRM, handle_confirm_request },
fcdfbd
         { PCMK__ATTRD_CMD_FLUSH, handle_flush_request },
fcdfbd
         { PCMK__ATTRD_CMD_PEER_REMOVE, handle_remove_request },
fcdfbd
         { PCMK__ATTRD_CMD_QUERY, handle_query_request },
fcdfbd
diff --git a/include/crm_internal.h b/include/crm_internal.h
fcdfbd
index 63a1726..f60e7b4 100644
fcdfbd
--- a/include/crm_internal.h
fcdfbd
+++ b/include/crm_internal.h
fcdfbd
@@ -108,6 +108,7 @@
fcdfbd
 #define PCMK__ATTRD_CMD_SYNC            "sync"
fcdfbd
 #define PCMK__ATTRD_CMD_SYNC_RESPONSE   "sync-response"
fcdfbd
 #define PCMK__ATTRD_CMD_CLEAR_FAILURE   "clear-failure"
fcdfbd
+#define PCMK__ATTRD_CMD_CONFIRM         "confirm"
fcdfbd
 
fcdfbd
 #define PCMK__CONTROLD_CMD_NODES        "list-nodes"
fcdfbd
 
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From dfb730e9ced9dc75886fda9452c584860573fe30 Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Wed, 26 Oct 2022 15:58:00 -0400
fcdfbd
Subject: [PATCH 18/26] Feature: daemons: Keep track of #attrd-protocol from
fcdfbd
 each peer.
fcdfbd
fcdfbd
This information can be used in the future when dealing with
fcdfbd
cluster-wide sync points to know which peers we are waiting on a reply
fcdfbd
from.
fcdfbd
---
fcdfbd
 daemons/attrd/attrd_corosync.c  |  3 +-
fcdfbd
 daemons/attrd/attrd_utils.c     | 60 ++++++++++++++++++++++++++++++---
fcdfbd
 daemons/attrd/pacemaker-attrd.h |  4 ++-
fcdfbd
 3 files changed, 60 insertions(+), 7 deletions(-)
fcdfbd
fcdfbd
diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
fcdfbd
index 1245d9c..6f88ab6 100644
fcdfbd
--- a/daemons/attrd/attrd_corosync.c
fcdfbd
+++ b/daemons/attrd/attrd_corosync.c
fcdfbd
@@ -268,6 +268,7 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da
fcdfbd
     // Remove votes from cluster nodes that leave, in case election in progress
fcdfbd
     if (gone && !is_remote) {
fcdfbd
         attrd_remove_voter(peer);
fcdfbd
+        attrd_remove_peer_protocol_ver(peer->uname);
fcdfbd
 
fcdfbd
     // Ensure remote nodes that come up are in the remote node cache
fcdfbd
     } else if (!gone && is_remote) {
fcdfbd
@@ -395,7 +396,7 @@ attrd_peer_update_one(const crm_node_t *peer, xmlNode *xml, bool filter)
fcdfbd
      * version, check to see if it's a new minimum version.
fcdfbd
      */
fcdfbd
     if (pcmk__str_eq(attr, CRM_ATTR_PROTOCOL, pcmk__str_none)) {
fcdfbd
-        attrd_update_minimum_protocol_ver(value);
fcdfbd
+        attrd_update_minimum_protocol_ver(peer->uname, value);
fcdfbd
     }
fcdfbd
 }
fcdfbd
 
fcdfbd
diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c
fcdfbd
index 00b879b..421faed 100644
fcdfbd
--- a/daemons/attrd/attrd_utils.c
fcdfbd
+++ b/daemons/attrd/attrd_utils.c
fcdfbd
@@ -29,6 +29,11 @@ static bool requesting_shutdown = false;
fcdfbd
 static bool shutting_down = false;
fcdfbd
 static GMainLoop *mloop = NULL;
fcdfbd
 
fcdfbd
+/* A hash table storing information on the protocol version of each peer attrd.
fcdfbd
+ * The key is the peer's uname, and the value is the protocol version number.
fcdfbd
+ */
fcdfbd
+GHashTable *peer_protocol_vers = NULL;
fcdfbd
+
fcdfbd
 /*!
fcdfbd
  * \internal
fcdfbd
  * \brief  Set requesting_shutdown state
fcdfbd
@@ -94,6 +99,10 @@ attrd_shutdown(int nsig)
fcdfbd
     mainloop_destroy_signal(SIGTRAP);
fcdfbd
 
fcdfbd
     attrd_free_waitlist();
fcdfbd
+    if (peer_protocol_vers != NULL) {
fcdfbd
+        g_hash_table_destroy(peer_protocol_vers);
fcdfbd
+        peer_protocol_vers = NULL;
fcdfbd
+    }
fcdfbd
 
fcdfbd
     if ((mloop == NULL) || !g_main_loop_is_running(mloop)) {
fcdfbd
         /* If there's no main loop active, just exit. This should be possible
fcdfbd
@@ -273,16 +282,57 @@ attrd_free_attribute(gpointer data)
fcdfbd
     }
fcdfbd
 }
fcdfbd
 
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief When a peer node leaves the cluster, stop tracking its protocol version.
fcdfbd
+ *
fcdfbd
+ * \param[in] host  The peer node's uname to be removed
fcdfbd
+ */
fcdfbd
+void
fcdfbd
+attrd_remove_peer_protocol_ver(const char *host)
fcdfbd
+{
fcdfbd
+    if (peer_protocol_vers != NULL) {
fcdfbd
+        g_hash_table_remove(peer_protocol_vers, host);
fcdfbd
+    }
fcdfbd
+}
fcdfbd
+
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief When a peer node broadcasts a message with its protocol version, keep
fcdfbd
+ *        track of that information.
fcdfbd
+ *
fcdfbd
+ * We keep track of each peer's protocol version so we know which peers to
fcdfbd
+ * expect confirmation messages from when handling cluster-wide sync points.
fcdfbd
+ * We additionally keep track of the lowest protocol version supported by all
fcdfbd
+ * peers so we know when we can send IPC messages containing more than one
fcdfbd
+ * request.
fcdfbd
+ *
fcdfbd
+ * \param[in] host  The peer node's uname to be tracked
fcdfbd
+ * \param[in] value The peer node's protocol version
fcdfbd
+ */
fcdfbd
 void
fcdfbd
-attrd_update_minimum_protocol_ver(const char *value)
fcdfbd
+attrd_update_minimum_protocol_ver(const char *host, const char *value)
fcdfbd
 {
fcdfbd
     int ver;
fcdfbd
 
fcdfbd
+    if (peer_protocol_vers == NULL) {
fcdfbd
+        peer_protocol_vers = pcmk__strkey_table(free, NULL);
fcdfbd
+    }
fcdfbd
+
fcdfbd
     pcmk__scan_min_int(value, &ver, 0);
fcdfbd
 
fcdfbd
-    if (ver > 0 && (minimum_protocol_version == -1 || ver < minimum_protocol_version)) {
fcdfbd
-        minimum_protocol_version = ver;
fcdfbd
-        crm_trace("Set minimum attrd protocol version to %d",
fcdfbd
-                  minimum_protocol_version);
fcdfbd
+    if (ver > 0) {
fcdfbd
+        char *host_name = strdup(host);
fcdfbd
+
fcdfbd
+        /* Record the peer attrd's protocol version. */
fcdfbd
+        CRM_ASSERT(host_name != NULL);
fcdfbd
+        g_hash_table_insert(peer_protocol_vers, host_name, GINT_TO_POINTER(ver));
fcdfbd
+
fcdfbd
+        /* If the protocol version is a new minimum, record it as such. */
fcdfbd
+        if (minimum_protocol_version == -1 || ver < minimum_protocol_version) {
fcdfbd
+            minimum_protocol_version = ver;
fcdfbd
+            crm_trace("Set minimum attrd protocol version to %d",
fcdfbd
+                      minimum_protocol_version);
fcdfbd
+        }
fcdfbd
     }
fcdfbd
 }
fcdfbd
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
fcdfbd
index 25f7c8a..302ef63 100644
fcdfbd
--- a/daemons/attrd/pacemaker-attrd.h
fcdfbd
+++ b/daemons/attrd/pacemaker-attrd.h
fcdfbd
@@ -145,6 +145,7 @@ typedef struct attribute_value_s {
fcdfbd
 
fcdfbd
 extern crm_cluster_t *attrd_cluster;
fcdfbd
 extern GHashTable *attributes;
fcdfbd
+extern GHashTable *peer_protocol_vers;
fcdfbd
 
fcdfbd
 #define CIB_OP_TIMEOUT_S 120
fcdfbd
 
fcdfbd
@@ -177,7 +178,8 @@ void attrd_write_attributes(bool all, bool ignore_delay);
fcdfbd
 void attrd_write_or_elect_attribute(attribute_t *a);
fcdfbd
 
fcdfbd
 extern int minimum_protocol_version;
fcdfbd
-void attrd_update_minimum_protocol_ver(const char *value);
fcdfbd
+void attrd_remove_peer_protocol_ver(const char *host);
fcdfbd
+void attrd_update_minimum_protocol_ver(const char *host, const char *value);
fcdfbd
 
fcdfbd
 mainloop_timer_t *attrd_add_timer(const char *id, int timeout_ms, attribute_t *attr);
fcdfbd
 
fcdfbd
-- 
fcdfbd
2.31.1
fcdfbd
fcdfbd
From 945f0fe51d3bf69c2cb1258b394f2f11b8996525 Mon Sep 17 00:00:00 2001
fcdfbd
From: Chris Lumens <clumens@redhat.com>
fcdfbd
Date: Thu, 27 Oct 2022 14:42:59 -0400
fcdfbd
Subject: [PATCH 19/26] Feature: daemons: Handle cluster-wide sync points in
fcdfbd
 attrd.
fcdfbd
fcdfbd
When an attrd receives an IPC request to update some value, record the
fcdfbd
protocol versions of all peer attrds.  Additionally register a function
fcdfbd
that will be called when all confirmations are received.
fcdfbd
fcdfbd
The originating IPC cilent (attrd_updater for instance) will sit there
fcdfbd
waiting for an ACK until its timeout is hit.
fcdfbd
fcdfbd
As each confirmation message comes back to attrd, mark it off the list
fcdfbd
of peers we are waiting on.  When no more peers are expected, call the
fcdfbd
previously registered function.
fcdfbd
fcdfbd
For attribute updates, this function just sends an ack back to
fcdfbd
attrd_updater.
fcdfbd
fcdfbd
Fixes T35
fcdfbd
---
fcdfbd
 daemons/attrd/attrd_corosync.c  |   1 +
fcdfbd
 daemons/attrd/attrd_ipc.c       |   4 +
fcdfbd
 daemons/attrd/attrd_messages.c  |  10 ++
fcdfbd
 daemons/attrd/attrd_sync.c      | 260 +++++++++++++++++++++++++++++++-
fcdfbd
 daemons/attrd/attrd_utils.c     |   2 +
fcdfbd
 daemons/attrd/pacemaker-attrd.h |   8 +
fcdfbd
 6 files changed, 281 insertions(+), 4 deletions(-)
fcdfbd
fcdfbd
diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
fcdfbd
index 6f88ab6..37701aa 100644
fcdfbd
--- a/daemons/attrd/attrd_corosync.c
fcdfbd
+++ b/daemons/attrd/attrd_corosync.c
fcdfbd
@@ -269,6 +269,7 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da
fcdfbd
     if (gone && !is_remote) {
fcdfbd
         attrd_remove_voter(peer);
fcdfbd
         attrd_remove_peer_protocol_ver(peer->uname);
fcdfbd
+        attrd_do_not_expect_from_peer(peer->uname);
fcdfbd
 
fcdfbd
     // Ensure remote nodes that come up are in the remote node cache
fcdfbd
     } else if (!gone && is_remote) {
fcdfbd
diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c
fcdfbd
index 0fc5e93..c70aa1b 100644
fcdfbd
--- a/daemons/attrd/attrd_ipc.c
fcdfbd
+++ b/daemons/attrd/attrd_ipc.c
fcdfbd
@@ -393,6 +393,7 @@ attrd_client_update(pcmk__request_t *request)
fcdfbd
          * the response ACK is not sent until this attrd broadcasts the update
fcdfbd
          * and receives its own confirmation back from all peers.
fcdfbd
          */
fcdfbd
+        attrd_expect_confirmations(request, attrd_cluster_sync_point_update);
fcdfbd
         attrd_send_message(NULL, xml, true); /* ends up at attrd_peer_message() */
fcdfbd
 
fcdfbd
     } else {
fcdfbd
@@ -456,6 +457,9 @@ attrd_ipc_closed(qb_ipcs_connection_t *c)
fcdfbd
         /* Remove the client from the sync point waitlist if it's present. */
fcdfbd
         attrd_remove_client_from_waitlist(client);
fcdfbd
 
fcdfbd
+        /* And no longer wait for confirmations from any peers. */
fcdfbd
+        attrd_do_not_wait_for_client(client);
fcdfbd
+
fcdfbd
         pcmk__free_client(client);
fcdfbd
     }
fcdfbd
 
fcdfbd
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
fcdfbd
index 9c792b2..f7b9c7c 100644
fcdfbd
--- a/daemons/attrd/attrd_messages.c
fcdfbd
+++ b/daemons/attrd/attrd_messages.c
fcdfbd
@@ -69,7 +69,17 @@ static xmlNode *
fcdfbd
 handle_confirm_request(pcmk__request_t *request)
fcdfbd
 {
fcdfbd
     if (request->peer != NULL) {
fcdfbd
+        int callid;
fcdfbd
+
fcdfbd
         crm_debug("Received confirmation from %s", request->peer);
fcdfbd
+
fcdfbd
+        if (crm_element_value_int(request->xml, XML_LRM_ATTR_CALLID, &callid) == -1) {
fcdfbd
+            pcmk__set_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
fcdfbd
+                             "Could not get callid from XML");
fcdfbd
+        } else {
fcdfbd
+            attrd_handle_confirmation(callid, request->peer);
fcdfbd
+        }
fcdfbd
+
fcdfbd
         pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
fcdfbd
         return NULL;
fcdfbd
     } else {
fcdfbd
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
fcdfbd
index e9690b5..d3d7108 100644
fcdfbd
--- a/daemons/attrd/attrd_sync.c
fcdfbd
+++ b/daemons/attrd/attrd_sync.c
fcdfbd
@@ -34,6 +34,51 @@ struct waitlist_node {
fcdfbd
     uint32_t flags;
fcdfbd
 };
fcdfbd
 
fcdfbd
+/* A hash table storing information on in-progress IPC requests that are awaiting
fcdfbd
+ * confirmations.  These requests are currently being processed by peer attrds and
fcdfbd
+ * we are waiting to receive confirmation messages from each peer indicating that
fcdfbd
+ * processing is complete.
fcdfbd
+ *
fcdfbd
+ * Multiple requests could be waiting on confirmations at the same time.
fcdfbd
+ *
fcdfbd
+ * The key is the unique callid for the IPC request, and the value is a
fcdfbd
+ * confirmation_action struct.
fcdfbd
+ */
fcdfbd
+static GHashTable *expected_confirmations = NULL;
fcdfbd
+
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief A structure describing a single IPC request that is awaiting confirmations
fcdfbd
+ */
fcdfbd
+struct confirmation_action {
fcdfbd
+    /*!
fcdfbd
+     * \brief A list of peer attrds that we are waiting to receive confirmation
fcdfbd
+     *        messages from
fcdfbd
+     *
fcdfbd
+     * This list is dynamic - as confirmations arrive from peer attrds, they will
fcdfbd
+     * be removed from this list.  When the list is empty, all peers have processed
fcdfbd
+     * the request and the associated confirmation action will be taken.
fcdfbd
+     */
fcdfbd
+    GList *respondents;
fcdfbd
+
fcdfbd
+    /*!
fcdfbd
+     * \brief A function to run when all confirmations have been received
fcdfbd
+     */
fcdfbd
+    attrd_confirmation_action_fn fn;
fcdfbd
+
fcdfbd
+    /*!
fcdfbd
+     * \brief Information required to construct and send a reply to the client
fcdfbd
+     */
fcdfbd
+    char *client_id;
fcdfbd
+    uint32_t ipc_id;
fcdfbd
+    uint32_t flags;
fcdfbd
+
fcdfbd
+    /*!
fcdfbd
+     * \brief The XML request containing the callid associated with this action
fcdfbd
+     */
fcdfbd
+    void *xml;
fcdfbd
+};
fcdfbd
+
fcdfbd
 static void
fcdfbd
 next_key(void)
fcdfbd
 {
fcdfbd
@@ -114,12 +159,13 @@ attrd_add_client_to_waitlist(pcmk__request_t *request)
fcdfbd
     wl->ipc_id = request->ipc_id;
fcdfbd
     wl->flags = request->flags;
fcdfbd
 
fcdfbd
-    crm_debug("Added client %s to waitlist for %s sync point",
fcdfbd
-              wl->client_id, sync_point_str(wl->sync_point));
fcdfbd
-
fcdfbd
     next_key();
fcdfbd
     pcmk__intkey_table_insert(waitlist, waitlist_client, wl);
fcdfbd
 
fcdfbd
+    crm_trace("Added client %s to waitlist for %s sync point",
fcdfbd
+              wl->client_id, sync_point_str(wl->sync_point));
fcdfbd
+    crm_trace("%d clients now on waitlist", g_hash_table_size(waitlist));
fcdfbd
+
fcdfbd
     /* And then add the key to the request XML so we can uniquely identify
fcdfbd
      * it when it comes time to issue the ACK.
fcdfbd
      */
fcdfbd
@@ -166,6 +212,7 @@ attrd_remove_client_from_waitlist(pcmk__client_t *client)
fcdfbd
 
fcdfbd
         if (wl->client_id == client->id) {
fcdfbd
             g_hash_table_iter_remove(&iter);
fcdfbd
+            crm_trace("%d clients now on waitlist", g_hash_table_size(waitlist));
fcdfbd
         }
fcdfbd
     }
fcdfbd
 }
fcdfbd
@@ -206,7 +253,7 @@ attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fcdfbd
             return;
fcdfbd
         }
fcdfbd
 
fcdfbd
-        crm_debug("Alerting client %s for reached %s sync point",
fcdfbd
+        crm_trace("Alerting client %s for reached %s sync point",
fcdfbd
                   wl->client_id, sync_point_str(wl->sync_point));
fcdfbd
 
fcdfbd
         client = pcmk__find_client_by_id(wl->client_id);
fcdfbd
@@ -218,9 +265,28 @@ attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
fcdfbd
 
fcdfbd
         /* And then remove the client so it doesn't get alerted again. */
fcdfbd
         pcmk__intkey_table_remove(waitlist, callid);
fcdfbd
+
fcdfbd
+        crm_trace("%d clients now on waitlist", g_hash_table_size(waitlist));
fcdfbd
     }
fcdfbd
 }
fcdfbd
 
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief Action to take when a cluster sync point is hit for a
fcdfbd
+ *        PCMK__ATTRD_CMD_UPDATE* message.
fcdfbd
+ *
fcdfbd
+ * \param[in] xml  The request that should be passed along to
fcdfbd
+ *                 attrd_ack_waitlist_clients.  This should be the original
fcdfbd
+ *                 IPC request containing the callid for this update message.
fcdfbd
+ */
fcdfbd
+int
fcdfbd
+attrd_cluster_sync_point_update(xmlNode *xml)
fcdfbd
+{
fcdfbd
+    crm_trace("Hit cluster sync point for attribute update");
fcdfbd
+    attrd_ack_waitlist_clients(attrd_sync_point_cluster, xml);
fcdfbd
+    return pcmk_rc_ok;
fcdfbd
+}
fcdfbd
+
fcdfbd
 /*!
fcdfbd
  * \internal
fcdfbd
  * \brief Return the sync point attribute for an IPC request
fcdfbd
@@ -268,3 +334,189 @@ attrd_request_has_sync_point(xmlNode *xml)
fcdfbd
 {
fcdfbd
     return attrd_request_sync_point(xml) != NULL;
fcdfbd
 }
fcdfbd
+
fcdfbd
+static void
fcdfbd
+free_action(gpointer data)
fcdfbd
+{
fcdfbd
+    struct confirmation_action *action = (struct confirmation_action *) data;
fcdfbd
+    g_list_free_full(action->respondents, free);
fcdfbd
+    free_xml(action->xml);
fcdfbd
+    free(action->client_id);
fcdfbd
+    free(action);
fcdfbd
+}
fcdfbd
+
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief When a peer disconnects from the cluster, no longer wait for its confirmation
fcdfbd
+ *        for any IPC action.  If this peer is the last one being waited on, this will
fcdfbd
+ *        trigger the confirmation action.
fcdfbd
+ *
fcdfbd
+ * \param[in] host   The disconnecting peer attrd's uname
fcdfbd
+ */
fcdfbd
+void
fcdfbd
+attrd_do_not_expect_from_peer(const char *host)
fcdfbd
+{
fcdfbd
+    GList *keys = g_hash_table_get_keys(expected_confirmations);
fcdfbd
+
fcdfbd
+    crm_trace("Removing peer %s from expected confirmations", host);
fcdfbd
+
fcdfbd
+    for (GList *node = keys; node != NULL; node = node->next) {
fcdfbd
+        int callid = *(int *) node->data;
fcdfbd
+        attrd_handle_confirmation(callid, host);
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    g_list_free(keys);
fcdfbd
+}
fcdfbd
+
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief When a client disconnects from the cluster, no longer wait on confirmations
fcdfbd
+ *        for it.  Because the peer attrds may still be processing the original IPC
fcdfbd
+ *        message, they may still send us confirmations.  However, we will take no
fcdfbd
+ *        action on them.
fcdfbd
+ *
fcdfbd
+ * \param[in] client    The disconnecting client
fcdfbd
+ */
fcdfbd
+void
fcdfbd
+attrd_do_not_wait_for_client(pcmk__client_t *client)
fcdfbd
+{
fcdfbd
+    GHashTableIter iter;
fcdfbd
+    gpointer value;
fcdfbd
+
fcdfbd
+    if (expected_confirmations == NULL) {
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    g_hash_table_iter_init(&iter, expected_confirmations);
fcdfbd
+
fcdfbd
+    while (g_hash_table_iter_next(&iter, NULL, &value)) {
fcdfbd
+        struct confirmation_action *action = (struct confirmation_action *) value;
fcdfbd
+
fcdfbd
+        if (pcmk__str_eq(action->client_id, client->id, pcmk__str_none)) {
fcdfbd
+            crm_trace("Removing client %s from expected confirmations", client->id);
fcdfbd
+            g_hash_table_iter_remove(&iter);
fcdfbd
+            crm_trace("%d requests now in expected confirmations table", g_hash_table_size(expected_confirmations));
fcdfbd
+            break;
fcdfbd
+        }
fcdfbd
+    }
fcdfbd
+}
fcdfbd
+
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief Register some action to be taken when IPC request confirmations are
fcdfbd
+ *        received
fcdfbd
+ *
fcdfbd
+ * When this function is called, a list of all peer attrds that support confirming
fcdfbd
+ * requests is generated.  As confirmations from these peer attrds are received,
fcdfbd
+ * they are removed from this list.  When the list is empty, the registered action
fcdfbd
+ * will be called.
fcdfbd
+ *
fcdfbd
+ * \note This function should always be called before attrd_send_message is called
fcdfbd
+ *       to broadcast to the peers to ensure that we know what replies we are
fcdfbd
+ *       waiting on.  Otherwise, it is possible the peer could finish and confirm
fcdfbd
+ *       before we know to expect it.
fcdfbd
+ *
fcdfbd
+ * \param[in] request The request that is awaiting confirmations
fcdfbd
+ * \param[in] fn      A function to be run after all confirmations are received
fcdfbd
+ */
fcdfbd
+void
fcdfbd
+attrd_expect_confirmations(pcmk__request_t *request, attrd_confirmation_action_fn fn)
fcdfbd
+{
fcdfbd
+    struct confirmation_action *action = NULL;
fcdfbd
+    GHashTableIter iter;
fcdfbd
+    gpointer host, ver;
fcdfbd
+    GList *respondents = NULL;
fcdfbd
+    int callid;
fcdfbd
+
fcdfbd
+    if (expected_confirmations == NULL) {
fcdfbd
+        expected_confirmations = pcmk__intkey_table((GDestroyNotify) free_action);
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    if (crm_element_value_int(request->xml, XML_LRM_ATTR_CALLID, &callid) == -1) {
fcdfbd
+        crm_err("Could not get callid from xml");
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    if (pcmk__intkey_table_lookup(expected_confirmations, callid)) {
fcdfbd
+        crm_err("Already waiting on confirmations for call id %d", callid);
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    g_hash_table_iter_init(&iter, peer_protocol_vers);
fcdfbd
+    while (g_hash_table_iter_next(&iter, &host, &ver)) {
fcdfbd
+        if (GPOINTER_TO_INT(ver) >= 5) {
fcdfbd
+            char *s = strdup((char *) host);
fcdfbd
+
fcdfbd
+            CRM_ASSERT(s != NULL);
fcdfbd
+            respondents = g_list_prepend(respondents, s);
fcdfbd
+        }
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    action = calloc(1, sizeof(struct confirmation_action));
fcdfbd
+    CRM_ASSERT(action != NULL);
fcdfbd
+
fcdfbd
+    action->respondents = respondents;
fcdfbd
+    action->fn = fn;
fcdfbd
+    action->xml = copy_xml(request->xml);
fcdfbd
+
fcdfbd
+    action->client_id = strdup(request->ipc_client->id);
fcdfbd
+    CRM_ASSERT(action->client_id != NULL);
fcdfbd
+
fcdfbd
+    action->ipc_id = request->ipc_id;
fcdfbd
+    action->flags = request->flags;
fcdfbd
+
fcdfbd
+    pcmk__intkey_table_insert(expected_confirmations, callid, action);
fcdfbd
+    crm_trace("Callid %d now waiting on %d confirmations", callid, g_list_length(respondents));
fcdfbd
+    crm_trace("%d requests now in expected confirmations table", g_hash_table_size(expected_confirmations));
fcdfbd
+}
fcdfbd
+
fcdfbd
+void
fcdfbd
+attrd_free_confirmations(void)
fcdfbd
+{
fcdfbd
+    if (expected_confirmations != NULL) {
fcdfbd
+        g_hash_table_destroy(expected_confirmations);
fcdfbd
+        expected_confirmations = NULL;
fcdfbd
+    }
fcdfbd
+}
fcdfbd
+
fcdfbd
+/*!
fcdfbd
+ * \internal
fcdfbd
+ * \brief Process a confirmation message from a peer attrd
fcdfbd
+ *
fcdfbd
+ * This function is called every time a PCMK__ATTRD_CMD_CONFIRM message is
fcdfbd
+ * received from a peer attrd.  If this is the last confirmation we are waiting
fcdfbd
+ * on for a given operation, the registered action will be called.
fcdfbd
+ *
fcdfbd
+ * \param[in] callid The unique callid for the XML IPC request
fcdfbd
+ * \param[in] host   The confirming peer attrd's uname
fcdfbd
+ */
fcdfbd
+void
fcdfbd
+attrd_handle_confirmation(int callid, const char *host)
fcdfbd
+{
fcdfbd
+    struct confirmation_action *action = NULL;
fcdfbd
+    GList *node = NULL;
fcdfbd
+
fcdfbd
+    if (expected_confirmations == NULL) {
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    action = pcmk__intkey_table_lookup(expected_confirmations, callid);
fcdfbd
+    if (action == NULL) {
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    node = g_list_find_custom(action->respondents, host, (GCompareFunc) strcasecmp);
fcdfbd
+
fcdfbd
+    if (node == NULL) {
fcdfbd
+        return;
fcdfbd
+    }
fcdfbd
+
fcdfbd
+    action->respondents = g_list_remove(action->respondents, node->data);
fcdfbd
+    crm_trace("Callid %d now waiting on %d confirmations", callid, g_list_length(action->respondents));
fcdfbd
+
fcdfbd
+    if (action->respondents == NULL) {
fcdfbd
+        action->fn(action->xml);
fcdfbd
+        pcmk__intkey_table_remove(expected_confirmations, callid);
fcdfbd
+        crm_trace("%d requests now in expected confirmations table", g_hash_table_size(expected_confirmations));
fcdfbd
+    }
fcdfbd
+}
fcdfbd
diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c
fcdfbd
index 421faed..f3a2059