Blob Blame History Raw
From 23d14e3515d226fee3ec9e0328f001f53597dad2 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 11:23:46 -0700
Subject: [PATCH 01/22] API: libpacemaker: pcmk_pacemakerd_status() ipc_name
 arg is now const

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 include/pacemaker.h                   | 11 ++++++-----
 include/pcmki/pcmki_cluster_queries.h |  3 ++-
 lib/pacemaker/pcmk_cluster_queries.c  | 17 +++++++++++++++--
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/include/pacemaker.h b/include/pacemaker.h
index 17c68e9..a76569a 100644
--- a/include/pacemaker.h
+++ b/include/pacemaker.h
@@ -107,15 +107,16 @@ int pcmk_designated_controller(xmlNodePtr *xml, unsigned int message_timeout_ms)
 void pcmk_free_injections(pcmk_injections_t *injections);
 
 /*!
- * \brief Get pacemakerd status
+ * \brief Get and output \p pacemakerd status
  *
- * \param[in,out] xml                The destination for the result, as an XML tree.
- * \param[in]     ipc_name           IPC name for request
- * \param[in]     message_timeout_ms Message timeout
+ * \param[in,out] xml                 Destination for the result, as an XML tree
+ * \param[in]     ipc_name            IPC name for request
+ * \param[in]     message_timeout_ms  Message timeout
  *
  * \return Standard Pacemaker return code
  */
-int pcmk_pacemakerd_status(xmlNodePtr *xml, char *ipc_name, unsigned int message_timeout_ms);
+int pcmk_pacemakerd_status(xmlNodePtr *xml, const char *ipc_name,
+                           unsigned int message_timeout_ms);
 
 /*!
  * \brief Calculate and output resource operation digests
diff --git a/include/pcmki/pcmki_cluster_queries.h b/include/pcmki/pcmki_cluster_queries.h
index 0a4c21c..9aea9a5 100644
--- a/include/pcmki/pcmki_cluster_queries.h
+++ b/include/pcmki/pcmki_cluster_queries.h
@@ -10,7 +10,8 @@
 
 int pcmk__controller_status(pcmk__output_t *out, char *dest_node, guint message_timeout_ms);
 int pcmk__designated_controller(pcmk__output_t *out, guint message_timeout_ms);
-int pcmk__pacemakerd_status(pcmk__output_t *out, char *ipc_name, guint message_timeout_ms);
+int pcmk__pacemakerd_status(pcmk__output_t *out, const char *ipc_name,
+                            guint message_timeout_ms);
 int pcmk__list_nodes(pcmk__output_t *out, char *node_types, gboolean BASH_EXPORT);
 
 #endif
diff --git a/lib/pacemaker/pcmk_cluster_queries.c b/lib/pacemaker/pcmk_cluster_queries.c
index c30a9b8..cac8ce0 100644
--- a/lib/pacemaker/pcmk_cluster_queries.c
+++ b/lib/pacemaker/pcmk_cluster_queries.c
@@ -358,8 +358,19 @@ pcmk_designated_controller(xmlNodePtr *xml, unsigned int message_timeout_ms)
     return rc;
 }
 
+/*!
+ * \internal
+ * \brief Get and output \p pacemakerd status
+ *
+ * \param[in,out] out                 Output object
+ * \param[in]     ipc_name            IPC name for request
+ * \param[in]     message_timeout_ms  Message timeout
+ *
+ * \return Standard Pacemaker return code
+ */
 int
-pcmk__pacemakerd_status(pcmk__output_t *out, char *ipc_name, guint message_timeout_ms)
+pcmk__pacemakerd_status(pcmk__output_t *out, const char *ipc_name,
+                        guint message_timeout_ms)
 {
     data_t data = {
         .out = out,
@@ -385,8 +396,10 @@ pcmk__pacemakerd_status(pcmk__output_t *out, char *ipc_name, guint message_timeo
     return data.rc;
 }
 
+// Documented in header
 int
-pcmk_pacemakerd_status(xmlNodePtr *xml, char *ipc_name, unsigned int message_timeout_ms)
+pcmk_pacemakerd_status(xmlNodePtr *xml, const char *ipc_name,
+                       unsigned int message_timeout_ms)
 {
     pcmk__output_t *out = NULL;
     int rc = pcmk_rc_ok;
-- 
2.31.1

From b15f4030020a8c0aa1cdb9e72a633adff02944bc Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 12:19:49 -0700
Subject: [PATCH 02/22] Feature: pacemakerd: New
 pcmk__pcmkd_state_enum2friendly() function

Given an enum pcmk_pacemakerd_state value, this function returns a
user-friendly string representation. This will be used in future
commits.

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 include/crm/common/ipc_internal.h |  3 +++
 lib/common/ipc_pacemakerd.c       | 33 +++++++++++++++++++++++++++++++
 tools/crm_mon.c                   | 14 ++++++-------
 3 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/include/crm/common/ipc_internal.h b/include/crm/common/ipc_internal.h
index 2a0c562..ebde808 100644
--- a/include/crm/common/ipc_internal.h
+++ b/include/crm/common/ipc_internal.h
@@ -29,6 +29,7 @@ extern "C" {
 
 #include <crm_config.h>             // US_AUTH_GETPEEREID
 #include <crm/common/ipc.h>
+#include <crm/common/ipc_pacemakerd.h>  // enum pcmk_pacemakerd_state
 #include <crm/common/mainloop.h>    // mainloop_io_t
 
 /* denotes "non yieldable PID" on FreeBSD, or actual PID1 in scenarios that
@@ -250,6 +251,8 @@ pcmk__ipc_sys_name(const char *ipc_name, const char *fallback)
     return ipc_name ? ipc_name : ((crm_system_name ? crm_system_name : fallback));
 }
 
+const char *pcmk__pcmkd_state_enum2friendly(enum pcmk_pacemakerd_state state);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/common/ipc_pacemakerd.c b/lib/common/ipc_pacemakerd.c
index 2bec0d1..3777f95 100644
--- a/lib/common/ipc_pacemakerd.c
+++ b/lib/common/ipc_pacemakerd.c
@@ -62,6 +62,39 @@ pcmk_pacemakerd_api_daemon_state_enum2text(
     return "invalid";
 }
 
+/*!
+ * \internal
+ * \brief Return a friendly string representation of a \p pacemakerd state
+ *
+ * \param[in] state  \p pacemakerd state
+ *
+ * \return A user-friendly string representation of \p state, or
+ *         <tt>"Invalid pacemakerd state"</tt>
+ */
+const char *
+pcmk__pcmkd_state_enum2friendly(enum pcmk_pacemakerd_state state)
+{
+    switch (state) {
+        case pcmk_pacemakerd_state_init:
+            return "Initializing pacemaker";
+        case pcmk_pacemakerd_state_starting_daemons:
+            return "Pacemaker daemons are starting";
+        case pcmk_pacemakerd_state_wait_for_ping:
+            return "Waiting for startup trigger from SBD";
+        case pcmk_pacemakerd_state_running:
+            return "Pacemaker is running";
+        case pcmk_pacemakerd_state_shutting_down:
+            return "Pacemaker daemons are shutting down";
+        case pcmk_pacemakerd_state_shutdown_complete:
+            /* Assuming pacemakerd won't process messages while in
+             * shutdown_complete state unless reporting to SBD
+             */
+            return "Pacemaker daemons are shut down (reporting to SBD)";
+        default:
+            return "Invalid pacemakerd state";
+    }
+}
+
 // \return Standard Pacemaker return code
 static int
 new_data(pcmk_ipc_api_t *api)
diff --git a/tools/crm_mon.c b/tools/crm_mon.c
index eaf79bd..e8cb709 100644
--- a/tools/crm_mon.c
+++ b/tools/crm_mon.c
@@ -951,26 +951,26 @@ pacemakerd_status(void)
                     rc = ENOTCONN;
                     if ((output_format == mon_output_console) ||
                         (output_format == mon_output_plain)) {
+
+                        const char *state_str = NULL;
+                        state_str = pcmk__pcmkd_state_enum2friendly(state);
                         switch (state) {
                             case pcmk_pacemakerd_state_running:
                                 rc = pcmk_rc_ok;
                                 break;
                             case pcmk_pacemakerd_state_starting_daemons:
-                                out->info(out,"Pacemaker daemons starting ...");
+                                out->info(out, "%s", state_str);
                                 break;
                             case pcmk_pacemakerd_state_wait_for_ping:
-                                out->info(out,"Waiting for startup-trigger from SBD ...");
+                                out->info(out, "%s", state_str);
                                 break;
                             case pcmk_pacemakerd_state_shutting_down:
-                                out->info(out,"Pacemaker daemons shutting down ...");
+                                out->info(out, "%s", state_str);
                                 /* try our luck maybe CIB is still accessible */
                                 rc = pcmk_rc_ok;
                                 break;
                             case pcmk_pacemakerd_state_shutdown_complete:
-                                /* assuming pacemakerd doesn't dispatch any pings after entering
-                                * that state unless it is waiting for SBD
-                                */
-                                out->info(out,"Pacemaker daemons shut down - reporting to SBD ...");
+                                out->info(out, "%s", state_str);
                                 break;
                             default:
                                 break;
-- 
2.31.1

From 7eb4fa59db667f1904b607cde8ed8b9caf7a46ed Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 12:51:40 -0700
Subject: [PATCH 03/22] Low: libcrmcommon: Check invalid time value in
 pacemakerd API reply

If the XML_ATTR_TSTAMP attribute is present but can't be parsed as an
integer, value_ll gets set to PCMK__PARSE_INT_DEFAULT (-1). This should
never happen, but just in case, we should convert a negative to 0 before
we cast to time_t, an unsigned type.

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 lib/common/ipc_pacemakerd.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/common/ipc_pacemakerd.c b/lib/common/ipc_pacemakerd.c
index 3777f95..562308c 100644
--- a/lib/common/ipc_pacemakerd.c
+++ b/lib/common/ipc_pacemakerd.c
@@ -211,7 +211,7 @@ dispatch(pcmk_ipc_api_t *api, xmlNode *reply)
         reply_data.data.ping.status =
             pcmk__str_eq(crm_element_value(msg_data, XML_PING_ATTR_STATUS), "ok",
                          pcmk__str_casei)?pcmk_rc_ok:pcmk_rc_error;
-        reply_data.data.ping.last_good = (time_t) value_ll;
+        reply_data.data.ping.last_good = (value_ll < 0)? 0 : (time_t) value_ll;
         reply_data.data.ping.sys_from = crm_element_value(msg_data,
                                             XML_PING_ATTR_SYSFROM);
     } else if (pcmk__str_eq(value, CRM_OP_QUIT, pcmk__str_none)) {
-- 
2.31.1

From 3169eaafce20e2a444c3b96755daf36dd7143242 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 15:01:12 -0700
Subject: [PATCH 04/22] Low: libpacemaker: Correct default for pinged_buf in
 pacemakerd_event_cb

Default should be NULL so that the last_updated default gets used
correctly in the pacemakerd-health message.

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 lib/pacemaker/pcmk_cluster_queries.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pacemaker/pcmk_cluster_queries.c b/lib/pacemaker/pcmk_cluster_queries.c
index cac8ce0..43b2b1f 100644
--- a/lib/pacemaker/pcmk_cluster_queries.c
+++ b/lib/pacemaker/pcmk_cluster_queries.c
@@ -229,7 +229,7 @@ pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
         (reply->data.ping.status == pcmk_rc_ok)?
             pcmk_pacemakerd_api_daemon_state_enum2text(
                 reply->data.ping.state):"query failed",
-        (reply->data.ping.status == pcmk_rc_ok)?pinged_buf:"");
+        (reply->data.ping.status == pcmk_rc_ok)? pinged_buf : NULL);
     data->rc = pcmk_rc_ok;
     crm_time_free(crm_when);
     free(pinged_buf);
-- 
2.31.1

From c6141eb0f47fc806a309f99ec52ccb274b134533 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 22:48:40 -0700
Subject: [PATCH 05/22] Refactor: libpacemaker: Improve return codes in
 pcmk__pacemakerd_status

Use pcmk_rc_ipc_unresponsive if we don't get a response from the API,
and EBADMSG if we get an bad reply or unexpected reply type.

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 lib/pacemaker/pcmk_cluster_queries.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/pacemaker/pcmk_cluster_queries.c b/lib/pacemaker/pcmk_cluster_queries.c
index 43b2b1f..9937e16 100644
--- a/lib/pacemaker/pcmk_cluster_queries.c
+++ b/lib/pacemaker/pcmk_cluster_queries.c
@@ -207,6 +207,7 @@ pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
         out->err(out, "error: Bad reply from pacemakerd: %s",
                 crm_exit_str(status));
         event_done(data, pacemakerd_api);
+        data->rc = EBADMSG;
         return;
     }
 
@@ -214,6 +215,7 @@ pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
         out->err(out, "error: Unknown reply type %d from pacemakerd",
                 reply->reply_type);
         event_done(data, pacemakerd_api);
+        data->rc = EBADMSG;
         return;
     }
 
@@ -375,7 +377,7 @@ pcmk__pacemakerd_status(pcmk__output_t *out, const char *ipc_name,
     data_t data = {
         .out = out,
         .mainloop = NULL,
-        .rc = pcmk_rc_ok,
+        .rc = pcmk_rc_ipc_unresponsive,
         .message_timer_id = 0,
         .message_timeout_ms = message_timeout_ms
     };
-- 
2.31.1

From df2e449a29fe3460b98403767a16be4d89ef3455 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 15:12:23 -0700
Subject: [PATCH 06/22] Feature: libpacemaker: pacemakerd-health message
 accepts state

Previously, the pacemakerd-health message accepted only a state string.
This made it difficult to use different state strings for different
output formats.

Now, the pacemakerd-health message accepts an enum pcmk_pacemakerd_state
value and an optional state string. If the state string is not set, then
the formatter function looks up an appropriate string representation for
the state. If the state string is set, it acts as an explicit override
and is used in place of a lookup.

Note that this will cause "invalid" to be printed instead of "<null>"
for quiet text outputs, and it will cause "Invalid pacemakerd state" to
be printed instead of "unknown state" for the default output.

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 lib/pacemaker/pcmk_cluster_queries.c | 40 +++++++++++---------
 lib/pacemaker/pcmk_output.c          | 56 +++++++++++++++++++++-------
 2 files changed, 65 insertions(+), 31 deletions(-)

diff --git a/lib/pacemaker/pcmk_cluster_queries.c b/lib/pacemaker/pcmk_cluster_queries.c
index 9937e16..3e36a12 100644
--- a/lib/pacemaker/pcmk_cluster_queries.c
+++ b/lib/pacemaker/pcmk_cluster_queries.c
@@ -180,9 +180,6 @@ pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
     pcmk__output_t *out = data->out;
     pcmk_pacemakerd_api_reply_t *reply = event_data;
 
-    crm_time_t *crm_when;
-    char *pinged_buf = NULL;
-
     switch (event_type) {
         case pcmk_ipc_event_disconnect:
             if (data->rc == ECONNRESET) { // Unexpected
@@ -220,22 +217,29 @@ pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
     }
 
     // Parse desired information from reply
-    crm_when = crm_time_new(NULL);
-    crm_time_set_timet(crm_when, &reply->data.ping.last_good);
-    pinged_buf = crm_time_as_string(crm_when,
-        crm_time_log_date | crm_time_log_timeofday |
-            crm_time_log_with_timezone);
-
-    out->message(out, "pacemakerd-health",
-        reply->data.ping.sys_from,
-        (reply->data.ping.status == pcmk_rc_ok)?
-            pcmk_pacemakerd_api_daemon_state_enum2text(
-                reply->data.ping.state):"query failed",
-        (reply->data.ping.status == pcmk_rc_ok)? pinged_buf : NULL);
+    if (reply->data.ping.status == pcmk_rc_ok) {
+        crm_time_t *when = crm_time_new(NULL);
+        char *when_s = NULL;
+
+        crm_time_set_timet(when, &reply->data.ping.last_good);
+        when_s = crm_time_as_string(when,
+                                    crm_time_log_date
+                                    |crm_time_log_timeofday
+                                    |crm_time_log_with_timezone);
+
+        out->message(out, "pacemakerd-health",
+                     reply->data.ping.sys_from, reply->data.ping.state, NULL,
+                     when_s);
+
+        crm_time_free(when);
+        free(when_s);
+
+    } else {
+        out->message(out, "pacemakerd-health",
+                     reply->data.ping.sys_from, reply->data.ping.state,
+                     "query failed", NULL);
+    }
     data->rc = pcmk_rc_ok;
-    crm_time_free(crm_when);
-    free(pinged_buf);
-
     event_done(data, pacemakerd_api);
 }
 
diff --git a/lib/pacemaker/pcmk_output.c b/lib/pacemaker/pcmk_output.c
index 9a522a3..edd4b82 100644
--- a/lib/pacemaker/pcmk_output.c
+++ b/lib/pacemaker/pcmk_output.c
@@ -627,36 +627,65 @@ health_xml(pcmk__output_t *out, va_list args)
     return pcmk_rc_ok;
 }
 
-PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *", "const char *", "const char *")
+PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *", "int", "const char *",
+                  "const char *")
 static int
-pacemakerd_health_text(pcmk__output_t *out, va_list args)
+pacemakerd_health(pcmk__output_t *out, va_list args)
 {
     const char *sys_from = va_arg(args, const char *);
-    const char *state = va_arg(args, const char *);
+    enum pcmk_pacemakerd_state state =
+        (enum pcmk_pacemakerd_state) va_arg(args, int);
+    const char *state_s = va_arg(args, const char *);
     const char *last_updated = va_arg(args, const char *);
 
+    if (state_s == NULL) {
+        state_s = pcmk__pcmkd_state_enum2friendly(state);
+    }
+    return out->info(out, "Status of %s: '%s' (last updated %s)",
+                     (!pcmk__str_empty(sys_from)) ? sys_from : "unknown node",
+                     state_s,
+                     (!pcmk__str_empty(last_updated)) ? last_updated : "at unknown time");
+}
+
+PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *", "int", "const char *",
+                  "const char *")
+static int
+pacemakerd_health_text(pcmk__output_t *out, va_list args)
+{
     if (!out->is_quiet(out)) {
-        return out->info(out, "Status of %s: '%s' %s %s", crm_str(sys_from),
-                         crm_str(state), (!pcmk__str_empty(last_updated))?
-                         "last updated":"", crm_str(last_updated));
+        return pacemakerd_health(out, args);
     } else {
-        pcmk__formatted_printf(out, "%s\n", crm_str(state));
+        const char *sys_from G_GNUC_UNUSED = va_arg(args, const char *);
+        enum pcmk_pacemakerd_state state =
+            (enum pcmk_pacemakerd_state) va_arg(args, int);
+        const char *state_s = va_arg(args, const char *);
+        const char *last_updated G_GNUC_UNUSED = va_arg(args, const char *);
+
+        if (state_s == NULL) {
+            state_s = pcmk_pacemakerd_api_daemon_state_enum2text(state);
+        }
+        pcmk__formatted_printf(out, "%s\n", state_s);
         return pcmk_rc_ok;
     }
-
-    return pcmk_rc_no_output;
 }
 
-PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *", "const char *", "const char *")
+PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *", "int", "const char *",
+                  "const char *")
 static int
 pacemakerd_health_xml(pcmk__output_t *out, va_list args)
 {
     const char *sys_from = va_arg(args, const char *);
-    const char *state = va_arg(args, const char *);
+    enum pcmk_pacemakerd_state state =
+        (enum pcmk_pacemakerd_state) va_arg(args, int);
+    const char *state_s = va_arg(args, const char *);
     const char *last_updated = va_arg(args, const char *);
 
+    if (state_s == NULL) {
+        state_s = pcmk_pacemakerd_api_daemon_state_enum2text(state);
+    }
+
     pcmk__output_create_xml_node(out, crm_str(sys_from),
-                                 "state", crm_str(state),
+                                 "state", state_s,
                                  "last_updated", crm_str(last_updated),
                                  NULL);
     return pcmk_rc_ok;
@@ -1899,7 +1928,8 @@ static pcmk__message_entry_t fmt_functions[] = {
     { "locations-list", "xml", locations_list_xml },
     { "node-action", "default", node_action },
     { "node-action", "xml", node_action_xml },
-    { "pacemakerd-health", "default", pacemakerd_health_text },
+    { "pacemakerd-health", "default", pacemakerd_health },
+    { "pacemakerd-health", "text", pacemakerd_health_text },
     { "pacemakerd-health", "xml", pacemakerd_health_xml },
     { "profile", "default", profile_default, },
     { "profile", "xml", profile_xml },
-- 
2.31.1

From 9bb521dc8b835641746095fe66b7a2137ce12c20 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 15:53:18 -0700
Subject: [PATCH 07/22] Feature: libpacemaker: pcmk__pacemakerd_status() can
 return pcmkd state

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 include/pcmki/pcmki_cluster_queries.h |  3 ++-
 lib/pacemaker/pcmk_cluster_queries.c  | 22 +++++++++++++++++-----
 tools/crmadmin.c                      |  3 ++-
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/include/pcmki/pcmki_cluster_queries.h b/include/pcmki/pcmki_cluster_queries.h
index 9aea9a5..702ab1f 100644
--- a/include/pcmki/pcmki_cluster_queries.h
+++ b/include/pcmki/pcmki_cluster_queries.h
@@ -11,7 +11,8 @@
 int pcmk__controller_status(pcmk__output_t *out, char *dest_node, guint message_timeout_ms);
 int pcmk__designated_controller(pcmk__output_t *out, guint message_timeout_ms);
 int pcmk__pacemakerd_status(pcmk__output_t *out, const char *ipc_name,
-                            guint message_timeout_ms);
+                            guint message_timeout_ms,
+                            enum pcmk_pacemakerd_state *state);
 int pcmk__list_nodes(pcmk__output_t *out, char *node_types, gboolean BASH_EXPORT);
 
 #endif
diff --git a/lib/pacemaker/pcmk_cluster_queries.c b/lib/pacemaker/pcmk_cluster_queries.c
index 3e36a12..5834ef0 100644
--- a/lib/pacemaker/pcmk_cluster_queries.c
+++ b/lib/pacemaker/pcmk_cluster_queries.c
@@ -36,6 +36,7 @@ typedef struct {
     int rc;
     guint message_timer_id;
     guint message_timeout_ms;
+    enum pcmk_pacemakerd_state pcmkd_state;
 } data_t;
 
 static void
@@ -217,6 +218,7 @@ pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
     }
 
     // Parse desired information from reply
+    data->pcmkd_state = reply->data.ping.state;
     if (reply->data.ping.status == pcmk_rc_ok) {
         crm_time_t *when = crm_time_new(NULL);
         char *when_s = NULL;
@@ -282,7 +284,8 @@ pcmk__controller_status(pcmk__output_t *out, char *dest_node, guint message_time
         .mainloop = NULL,
         .rc = pcmk_rc_ok,
         .message_timer_id = 0,
-        .message_timeout_ms = message_timeout_ms
+        .message_timeout_ms = message_timeout_ms,
+        .pcmkd_state = pcmk_pacemakerd_state_invalid,
     };
     pcmk_ipc_api_t *controld_api = ipc_connect(&data, pcmk_ipc_controld, controller_status_event_cb);
 
@@ -327,7 +330,8 @@ pcmk__designated_controller(pcmk__output_t *out, guint message_timeout_ms)
         .mainloop = NULL,
         .rc = pcmk_rc_ok,
         .message_timer_id = 0,
-        .message_timeout_ms = message_timeout_ms
+        .message_timeout_ms = message_timeout_ms,
+        .pcmkd_state = pcmk_pacemakerd_state_invalid,
     };
     pcmk_ipc_api_t *controld_api = ipc_connect(&data, pcmk_ipc_controld, designated_controller_event_cb);
 
@@ -371,19 +375,23 @@ pcmk_designated_controller(xmlNodePtr *xml, unsigned int message_timeout_ms)
  * \param[in,out] out                 Output object
  * \param[in]     ipc_name            IPC name for request
  * \param[in]     message_timeout_ms  Message timeout
+ * \param[out]    state               Where to store the \p pacemakerd state, if
+ *                                    not \p NULL
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk__pacemakerd_status(pcmk__output_t *out, const char *ipc_name,
-                        guint message_timeout_ms)
+                        guint message_timeout_ms,
+                        enum pcmk_pacemakerd_state *state)
 {
     data_t data = {
         .out = out,
         .mainloop = NULL,
         .rc = pcmk_rc_ipc_unresponsive,
         .message_timer_id = 0,
-        .message_timeout_ms = message_timeout_ms
+        .message_timeout_ms = message_timeout_ms,
+        .pcmkd_state = pcmk_pacemakerd_state_invalid,
     };
     pcmk_ipc_api_t *pacemakerd_api = ipc_connect(&data, pcmk_ipc_pacemakerd, pacemakerd_event_cb);
 
@@ -399,6 +407,9 @@ pcmk__pacemakerd_status(pcmk__output_t *out, const char *ipc_name,
         pcmk_free_ipc_api(pacemakerd_api);
     }
 
+    if (state != NULL) {
+        *state = data.pcmkd_state;
+    }
     return data.rc;
 }
 
@@ -417,7 +428,8 @@ pcmk_pacemakerd_status(xmlNodePtr *xml, const char *ipc_name,
 
     pcmk__register_lib_messages(out);
 
-    rc = pcmk__pacemakerd_status(out, ipc_name, (guint) message_timeout_ms);
+    rc = pcmk__pacemakerd_status(out, ipc_name, (guint) message_timeout_ms,
+                                 NULL);
     pcmk__out_epilogue(out, xml, rc);
     return rc;
 }
diff --git a/tools/crmadmin.c b/tools/crmadmin.c
index 169289f..f4c2783 100644
--- a/tools/crmadmin.c
+++ b/tools/crmadmin.c
@@ -238,7 +238,8 @@ main(int argc, char **argv)
             rc = pcmk__controller_status(out, options.optarg, options.timeout);
             break;
         case cmd_pacemakerd_health:
-            rc = pcmk__pacemakerd_status(out, options.ipc_name, options.timeout);
+            rc = pcmk__pacemakerd_status(out, options.ipc_name, options.timeout,
+                                         NULL);
             break;
         case cmd_list_nodes:
             rc = pcmk__list_nodes(out, options.optarg, options.BASH_EXPORT);
-- 
2.31.1

From 4841c22f9a7cc927e87007c9691e2c239f035a58 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 16:50:06 -0700
Subject: [PATCH 08/22] Fix: libpacemaker: Memory leak in
 pcmk_cluster_queries.c:ipc_connect()

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 lib/pacemaker/pcmk_cluster_queries.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/pacemaker/pcmk_cluster_queries.c b/lib/pacemaker/pcmk_cluster_queries.c
index 5834ef0..00a809d 100644
--- a/lib/pacemaker/pcmk_cluster_queries.c
+++ b/lib/pacemaker/pcmk_cluster_queries.c
@@ -270,6 +270,7 @@ ipc_connect(data_t *data, enum pcmk_ipc_server server, pcmk_ipc_callback_t cb)
                 pcmk_ipc_name(api, true),
                 pcmk_rc_str(rc));
         data->rc = rc;
+        pcmk_free_ipc_api(api);
         return NULL;
     }
 
-- 
2.31.1

From 8e202448c47ad0ddc148b2e0514ef98b4847fa6e Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 18:25:35 -0700
Subject: [PATCH 09/22] Doc: libpe_status: Replace old funcname in
 pe__build_rsc_list() comment

build_uname_list -> pe__build_node_name_list()

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 lib/pengine/utils.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index 77111a6..1a4eb3e 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -2551,9 +2551,9 @@ pe__build_rsc_list(pe_working_set_t *data_set, const char *s) {
                 resources = g_list_prepend(resources, strdup(rsc_printable_id(rsc)));
             }
         } else {
-            /* The given string was not a valid resource name.  It's either
-             * a tag or it's a typo or something.  See build_uname_list for
-             * more detail.
+            /* The given string was not a valid resource name. It's a tag or a
+             * typo or something. See pe__build_node_name_list() for more
+             * detail.
              */
             resources = pe__rscs_with_tag(data_set, s);
         }
-- 
2.31.1

From 0c412f49d607a8f60790b13e75d8c7b3a8c6c1d9 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 18:38:06 -0700
Subject: [PATCH 10/22] Refactor: libpacemaker: Clarify pointer arguments in
 pcmk_status.c

Make only_node and only_rsc arguments const. Add doxygen blocks. Change
"st" argument to "stonith".

This is not comprehensive. We're updating pcmk__status() because we're
about to add a new argument, and pcmk__output_cluster_status() because
it shares most of its arguments with pcmk__status().

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 include/pcmki/pcmki_status.h | 18 ++++---
 lib/pacemaker/pcmk_status.c  | 93 +++++++++++++++++++++++++++++-------
 2 files changed, 87 insertions(+), 24 deletions(-)

diff --git a/include/pcmki/pcmki_status.h b/include/pcmki/pcmki_status.h
index 6614fe4..2bbd099 100644
--- a/include/pcmki/pcmki_status.h
+++ b/include/pcmki/pcmki_status.h
@@ -38,15 +38,19 @@ extern "C" {
  */
 int pcmk__output_simple_status(pcmk__output_t *out, pe_working_set_t *data_set);
 
-int pcmk__output_cluster_status(pcmk__output_t *out, stonith_t *st, cib_t *cib,
-                                xmlNode *current_cib, enum pcmk__fence_history fence_history,
-                                uint32_t show, uint32_t show_opts, char *only_node,
-                                char *only_rsc, char *neg_location_prefix,
+int pcmk__output_cluster_status(pcmk__output_t *out, stonith_t *stonith,
+                                cib_t *cib, xmlNode *current_cib,
+                                enum pcmk__fence_history fence_history,
+                                uint32_t show, uint32_t show_opts,
+                                const char *only_node, const char *only_rsc,
+                                const char *neg_location_prefix,
                                 bool simple_output);
 
-int pcmk__status(pcmk__output_t *out, cib_t *cib, enum pcmk__fence_history fence_history,
-                 uint32_t show, uint32_t show_opts, char *only_node, char *only_rsc,
-                 char *neg_location_prefix, bool simple_output);
+int pcmk__status(pcmk__output_t *out, cib_t *cib,
+                 enum pcmk__fence_history fence_history, uint32_t show,
+                 uint32_t show_opts, const char *only_node,
+                 const char *only_rsc, const char *neg_location_prefix,
+                 bool simple_output);
 
 #ifdef __cplusplus
 }
diff --git a/lib/pacemaker/pcmk_status.c b/lib/pacemaker/pcmk_status.c
index 12136ea..1bf0172 100644
--- a/lib/pacemaker/pcmk_status.c
+++ b/lib/pacemaker/pcmk_status.c
@@ -135,11 +135,38 @@ pacemakerd_status(pcmk__output_t *out)
     return rc;
 }
 
+/*!
+ * \internal
+ * \brief Output the cluster status given a fencer and CIB connection
+ *
+ * \param[in,out] out                  Output object
+ * \param[in,out] stonith              Fencer connection
+ * \param[in,out] cib                  CIB connection
+ * \param[in]     current_cib          Current CIB XML
+ * \param[in]     fence_history        How much of the fencing history to output
+ * \param[in]     show                 Group of \p pcmk_section_e flags
+ * \param[in]     show_opts            Group of \p pcmk_show_opt_e flags
+ * \param[in]     only_node            If a node name or tag, include only the
+ *                                     matching node(s) (if any) in the output.
+ *                                     If \p "*" or \p NULL, include all nodes
+ *                                     in the output.
+ * \param[in]     only_rsc             If a resource ID or tag, include only the
+ *                                     matching resource(s) (if any) in the
+ *                                     output. If \p "*" or \p NULL, include all
+ *                                     resources in the output.
+ * \param[in]     neg_location_prefix  Prefix denoting a ban in a constraint ID
+ * \param[in]     simple_output        Whether to use a simple output format.
+ *                                     Note: This is for use by \p crm_mon only
+ *                                     and is planned to be deprecated.
+ *
+ * \return Standard Pacemaker return code
+ */
 int
-pcmk__output_cluster_status(pcmk__output_t *out, stonith_t *st, cib_t *cib,
+pcmk__output_cluster_status(pcmk__output_t *out, stonith_t *stonith, cib_t *cib,
                             xmlNode *current_cib, enum pcmk__fence_history fence_history,
-                            uint32_t show, uint32_t show_opts, char *only_node,
-                            char *only_rsc, char *neg_location_prefix, bool simple_output)
+                            uint32_t show, uint32_t show_opts,
+                            const char *only_node, const char *only_rsc,
+                            const char *neg_location_prefix, bool simple_output)
 {
     xmlNode *cib_copy = copy_xml(current_cib);
     stonith_history_t *stonith_history = NULL;
@@ -159,7 +186,8 @@ pcmk__output_cluster_status(pcmk__output_t *out, stonith_t *st, cib_t *cib,
 
     /* get the stonith-history if there is evidence we need it */
     if (fence_history != pcmk__fence_history_none) {
-        history_rc = pcmk__get_fencing_history(st, &stonith_history, fence_history);
+        history_rc = pcmk__get_fencing_history(stonith, &stonith_history,
+                                               fence_history);
     }
 
     data_set = pe_new_working_set();
@@ -235,14 +263,43 @@ pcmk_status(xmlNodePtr *xml)
     return rc;
 }
 
+/*!
+ * \internal
+ * \brief Query and output the cluster status
+ *
+ * The operation is considered a success if we're able to get the \p pacemakerd
+ * state. If possible, we'll also try to connect to the fencer and CIB and
+ * output their respective status information.
+ *
+ * \param[in,out] out                  Output object
+ * \param[in,out] cib                  CIB connection
+ * \param[in]     fence_history        How much of the fencing history to output
+ * \param[in]     show                 Group of \p pcmk_section_e flags
+ * \param[in]     show_opts            Group of \p pcmk_show_opt_e flags
+ * \param[in]     only_node            If a node name or tag, include only the
+ *                                     matching node(s) (if any) in the output.
+ *                                     If \p "*" or \p NULL, include all nodes
+ *                                     in the output.
+ * \param[in]     only_rsc             If a resource ID or tag, include only the
+ *                                     matching resource(s) (if any) in the
+ *                                     output. If \p "*" or \p NULL, include all
+ *                                     resources in the output.
+ * \param[in]     neg_location_prefix  Prefix denoting a ban in a constraint ID
+ * \param[in]     simple_output        Whether to use a simple output format.
+ *                                     Note: This is for use by \p crm_mon only
+ *                                     and is planned to be deprecated.
+ *
+ * \return Standard Pacemaker return code
+ */
 int
-pcmk__status(pcmk__output_t *out, cib_t *cib, enum pcmk__fence_history fence_history,
-             uint32_t show, uint32_t show_opts, char *only_node, char *only_rsc,
-             char *neg_location_prefix, bool simple_output)
+pcmk__status(pcmk__output_t *out, cib_t *cib,
+             enum pcmk__fence_history fence_history, uint32_t show,
+             uint32_t show_opts, const char *only_node, const char *only_rsc,
+             const char *neg_location_prefix, bool simple_output)
 {
     xmlNode *current_cib = NULL;
     int rc = pcmk_rc_ok;
-    stonith_t *st = NULL;
+    stonith_t *stonith = NULL;
 
     if (cib == NULL) {
         return ENOTCONN;
@@ -261,9 +318,9 @@ pcmk__status(pcmk__output_t *out, cib_t *cib, enum pcmk__fence_history fence_his
     }
 
     if (fence_history != pcmk__fence_history_none && cib->variant == cib_native) {
-        st = fencing_connect();
+        stonith = fencing_connect();
 
-        if (st == NULL) {
+        if (stonith == NULL) {
             return ENOTCONN;
         }
     }
@@ -273,17 +330,19 @@ pcmk__status(pcmk__output_t *out, cib_t *cib, enum pcmk__fence_history fence_his
         goto done;
     }
 
-    rc = pcmk__output_cluster_status(out, st, cib, current_cib, fence_history, show, show_opts,
-                                     only_node, only_rsc, neg_location_prefix, simple_output);
+    rc = pcmk__output_cluster_status(out, stonith, cib, current_cib,
+                                     fence_history, show, show_opts, only_node,
+                                     only_rsc, neg_location_prefix,
+                                     simple_output);
 
 done:
-    if (st != NULL) {
-        if (st->state != stonith_disconnected) {
-            st->cmds->remove_notification(st, NULL);
-            st->cmds->disconnect(st);
+    if (stonith != NULL) {
+        if (stonith->state != stonith_disconnected) {
+            stonith->cmds->remove_notification(stonith, NULL);
+            stonith->cmds->disconnect(stonith);
         }
 
-        stonith_api_delete(st);
+        stonith_api_delete(stonith);
     }
 
     return rc;
-- 
2.31.1

From 8384af058c47a46cd10a070f90f6dc0bd1b12045 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 22:22:49 -0700
Subject: [PATCH 11/22] Feature: libpacemaker: HTML formatter for
 pacemakerd-health message

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 lib/pacemaker/pcmk_output.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/lib/pacemaker/pcmk_output.c b/lib/pacemaker/pcmk_output.c
index edd4b82..c088a6a 100644
--- a/lib/pacemaker/pcmk_output.c
+++ b/lib/pacemaker/pcmk_output.c
@@ -646,6 +646,32 @@ pacemakerd_health(pcmk__output_t *out, va_list args)
                      state_s,
                      (!pcmk__str_empty(last_updated)) ? last_updated : "at unknown time");
 }
+ 
+PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *", "int", "const char *",
+                  "const char *")
+static int
+pacemakerd_health_html(pcmk__output_t *out, va_list args)
+{
+    const char *sys_from = va_arg(args, const char *);
+    enum pcmk_pacemakerd_state state =
+        (enum pcmk_pacemakerd_state) va_arg(args, int);
+    const char *state_s = va_arg(args, const char *);
+    const char *last_updated = va_arg(args, const char *);
+    char *msg = NULL;
+
+    if (state_s == NULL) {
+        state_s = pcmk__pcmkd_state_enum2friendly(state);
+    }
+
+    msg = crm_strdup_printf("Status of %s: '%s' (last updated %s)",
+                            (!pcmk__str_empty(sys_from)) ? sys_from : "unknown node",
+                            state_s,
+                            (!pcmk__str_empty(last_updated)) ? last_updated : "at unknown time");
+    pcmk__output_create_html_node(out, "li", NULL, NULL, msg);
+
+    free(msg);
+    return pcmk_rc_ok;
+}
 
 PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *", "int", "const char *",
                   "const char *")
@@ -1929,6 +1955,7 @@ static pcmk__message_entry_t fmt_functions[] = {
     { "node-action", "default", node_action },
     { "node-action", "xml", node_action_xml },
     { "pacemakerd-health", "default", pacemakerd_health },
+    { "pacemakerd-health", "html", pacemakerd_health_html },
     { "pacemakerd-health", "text", pacemakerd_health_text },
     { "pacemakerd-health", "xml", pacemakerd_health_xml },
     { "profile", "default", profile_default, },
-- 
2.31.1

From ec6a28bf64d23107c81d473c02038c29b17f2917 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Sat, 3 Sep 2022 21:40:04 -0700
Subject: [PATCH 12/22] Low: schemas: Copy some API schemas in preparation for
 changes

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 include/crm/common/output_internal.h |   2 +-
 xml/api/command-output-2.23.rng      |  26 +++
 xml/api/crm_resource-2.23.rng        | 288 +++++++++++++++++++++++++++
 xml/api/stonith_admin-2.23.rng       |  52 +++++
 4 files changed, 367 insertions(+), 1 deletion(-)
 create mode 100644 xml/api/command-output-2.23.rng
 create mode 100644 xml/api/crm_resource-2.23.rng
 create mode 100644 xml/api/stonith_admin-2.23.rng

diff --git a/include/crm/common/output_internal.h b/include/crm/common/output_internal.h
index 24f5b2c..1e71e13 100644
--- a/include/crm/common/output_internal.h
+++ b/include/crm/common/output_internal.h
@@ -28,7 +28,7 @@ extern "C" {
  */
 
 
-#  define PCMK__API_VERSION "2.22"
+#  define PCMK__API_VERSION "2.23"
 
 #if defined(PCMK__WITH_ATTRIBUTE_OUTPUT_ARGS)
 #  define PCMK__OUTPUT_ARGS(ARGS...) __attribute__((output_args(ARGS)))
diff --git a/xml/api/command-output-2.23.rng b/xml/api/command-output-2.23.rng
new file mode 100644
index 0000000..710c134
--- /dev/null
+++ b/xml/api/command-output-2.23.rng
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="command-output" />
+    </start>
+
+    <define name="command-output">
+        <element name="command">
+            <attribute name="code"> <data type="integer" /> </attribute>
+            <optional>
+                <element name="output">
+                    <attribute name="source"><value>stdout</value></attribute>
+                    <text />
+                </element>
+            </optional>
+            <optional>
+                <element name="output">
+                    <attribute name="source"><value>stderr</value></attribute>
+                    <text />
+                </element>
+            </optional>
+        </element>
+    </define>
+</grammar>
diff --git a/xml/api/crm_resource-2.23.rng b/xml/api/crm_resource-2.23.rng
new file mode 100644
index 0000000..8a46675
--- /dev/null
+++ b/xml/api/crm_resource-2.23.rng
@@ -0,0 +1,288 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-crm-resource"/>
+    </start>
+
+    <define name="element-crm-resource">
+        <choice>
+            <ref name="agents-list" />
+            <ref name="alternatives-list" />
+            <ref name="constraints-list" />
+            <externalRef href="generic-list-2.4.rng"/>
+            <element name="metadata"> <text/> </element>
+            <ref name="locate-list" />
+            <ref name="operations-list" />
+            <ref name="providers-list" />
+            <ref name="reasons-list" />
+            <ref name="resource-check" />
+            <ref name="resource-config" />
+            <ref name="resources-list" />
+            <ref name="resource-agent-action" />
+        </choice>
+    </define>
+
+    <define name="agents-list">
+        <element name="agents">
+            <attribute name="standard"> <text/> </attribute>
+            <optional>
+                <attribute name="provider"> <text/> </attribute>
+            </optional>
+            <zeroOrMore>
+                <element name="agent"> <text/> </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="alternatives-list">
+        <element name="providers">
+            <attribute name="for"> <text/> </attribute>
+            <zeroOrMore>
+                <element name="provider"> <text/> </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="constraints-list">
+        <element name="constraints">
+            <interleave>
+                <zeroOrMore>
+                    <ref name="rsc-location" />
+                </zeroOrMore>
+                <zeroOrMore>
+                    <ref name="rsc-colocation" />
+                </zeroOrMore>
+            </interleave>
+        </element>
+    </define>
+
+    <define name="locate-list">
+        <element name="nodes">
+            <attribute name="resource"> <text/> </attribute>
+            <zeroOrMore>
+                <element name="node">
+                    <optional>
+                        <attribute name="state"><value>promoted</value></attribute>
+                    </optional>
+                    <text/>
+                </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="rsc-location">
+        <element name="rsc_location">
+            <attribute name="node"> <text/> </attribute>
+            <attribute name="rsc"> <text/> </attribute>
+            <attribute name="id"> <text/> </attribute>
+            <externalRef href="../score.rng"/>
+        </element>
+    </define>
+
+    <define name="operations-list">
+        <element name="operations">
+            <oneOrMore>
+                <ref name="element-operation-list" />
+            </oneOrMore>
+        </element>
+    </define>
+
+    <define name="providers-list">
+        <element name="providers">
+            <attribute name="standard"> <value>ocf</value> </attribute>
+            <optional>
+                <attribute name="agent"> <text/> </attribute>
+            </optional>
+            <zeroOrMore>
+                <element name="provider"> <text/> </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="reasons-list">
+        <element name="reason">
+            <!-- set only when resource and node are both specified -->
+            <optional>
+                <attribute name="running_on"> <text/> </attribute>
+            </optional>
+
+            <!-- set only when only a resource is specified -->
+            <optional>
+                <attribute name="running"> <data type="boolean"/> </attribute>
+            </optional>
+
+            <choice>
+                <ref name="reasons-with-no-resource"/>
+                <ref name="resource-check"/>
+            </choice>
+        </element>
+    </define>
+
+    <define name="reasons-with-no-resource">
+        <element name="resources">
+            <zeroOrMore>
+                <element name="resource">
+                    <attribute name="id"> <text/> </attribute>
+                    <attribute name="running"> <data type="boolean"/> </attribute>
+                    <optional>
+                        <attribute name="host"> <text/> </attribute>
+                    </optional>
+                    <ref name="resource-check"/>
+                </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="resource-config">
+        <element name="resource_config">
+            <externalRef href="resources-2.4.rng" />
+            <element name="xml"> <text/> </element>
+        </element>
+    </define>
+
+    <define name="resource-check">
+        <element name="check">
+            <attribute name="id"> <text/> </attribute>
+            <optional>
+                <choice>
+                    <attribute name="remain_stopped"><value>true</value></attribute>
+                    <attribute name="promotable"><value>false</value></attribute>
+                </choice>
+            </optional>
+            <optional>
+                <attribute name="unmanaged"><value>true</value></attribute>
+            </optional>
+            <optional>
+                <attribute name="locked-to"> <text/> </attribute>
+            </optional>
+            <optional>
+                <attribute name="unhealthy"><value>true</value></attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="resources-list">
+        <element name="resources">
+            <zeroOrMore>
+                <externalRef href="resources-2.4.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="rsc-colocation">
+        <element name="rsc_colocation">
+            <attribute name="id"> <text/> </attribute>
+            <attribute name="rsc"> <text/> </attribute>
+            <attribute name="with-rsc"> <text/> </attribute>
+            <externalRef href="../score.rng"/>
+            <optional>
+                <attribute name="node-attribute"> <text/> </attribute>
+            </optional>
+            <optional>
+                <attribute name="rsc-role">
+                    <ref name="attribute-roles"/>
+                </attribute>
+            </optional>
+            <optional>
+                <attribute name="with-rsc-role">
+                    <ref name="attribute-roles"/>
+                </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="element-operation-list">
+        <element name="operation">
+            <optional>
+                <group>
+                    <attribute name="rsc"> <text/> </attribute>
+                    <attribute name="agent"> <text/> </attribute>
+                </group>
+            </optional>
+            <attribute name="op"> <text/> </attribute>
+            <attribute name="node"> <text/> </attribute>
+            <attribute name="call"> <data type="integer" /> </attribute>
+            <attribute name="rc"> <data type="nonNegativeInteger" /> </attribute>
+            <optional>
+                <attribute name="last-rc-change"> <text/> </attribute>
+                <attribute name="exec-time"> <data type="nonNegativeInteger" /> </attribute>
+            </optional>
+            <attribute name="status"> <text/> </attribute>
+        </element>
+    </define>
+
+    <define name="resource-agent-action">
+        <element name="resource-agent-action">
+            <attribute name="action"> <text/> </attribute>
+            <optional>
+                <attribute name="rsc"> <text/> </attribute>
+            </optional>
+            <attribute name="class"> <text/> </attribute>
+            <attribute name="type"> <text/> </attribute>
+            <optional>
+                <attribute name="provider"> <text/> </attribute>
+            </optional>
+            <optional>
+                <ref name="overrides-list"/>
+            </optional>
+            <ref name="agent-status"/>
+            <optional>
+                <choice>
+                    <element name="command">
+                        <text />
+                    </element>
+                    <externalRef href="command-output-1.0.rng"/>
+                </choice>
+            </optional>
+        </element>
+    </define>
+
+    <define name="overrides-list">
+        <element name="overrides">
+            <zeroOrMore>
+                <element name="override">
+                    <optional>
+                        <attribute name="rsc"> <text/> </attribute>
+                    </optional>
+                    <attribute name="name"> <text/> </attribute>
+                    <attribute name="value"> <text/> </attribute>
+                </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="agent-status">
+        <element name="agent-status">
+            <attribute name="code"> <data type="integer" /> </attribute>
+            <optional>
+                <attribute name="message"> <text/> </attribute>
+            </optional>
+            <optional>
+                <attribute name="execution_code"> <data type="integer" /> </attribute>
+            </optional>
+            <optional>
+                <attribute name="execution_message"> <text/> </attribute>
+            </optional>
+            <optional>
+                <attribute name="reason"> <text/> </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="attribute-roles">
+        <choice>
+            <value>Stopped</value>
+            <value>Started</value>
+            <value>Promoted</value>
+            <value>Unpromoted</value>
+
+            <!-- These synonyms for Promoted/Unpromoted are allowed for
+                 backward compatibility with output from older Pacemaker
+                 versions that used them -->
+            <value>Master</value>
+            <value>Slave</value>
+        </choice>
+    </define>
+</grammar>
diff --git a/xml/api/stonith_admin-2.23.rng b/xml/api/stonith_admin-2.23.rng
new file mode 100644
index 0000000..b55fae9
--- /dev/null
+++ b/xml/api/stonith_admin-2.23.rng
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-stonith-admin"/>
+    </start>
+
+    <define name="element-stonith-admin">
+        <choice>
+            <ref name="stonith-admin-list" />
+            <ref name="element-last-fenced" />
+            <ref name="element-validation" />
+            <element name="metadata"> <text /> </element>
+        </choice>
+    </define>
+
+    <define name="stonith-admin-list">
+        <optional>
+            <element name="list">
+                <attribute name="name"> <text /> </attribute>
+                <attribute name="count"> <data type="nonNegativeInteger" /> </attribute>
+                <choice>
+                    <empty/>
+                    <oneOrMore>
+                        <externalRef href="item-1.1.rng"/>
+                    </oneOrMore>
+                    <oneOrMore>
+                        <externalRef href="fence-event-2.15.rng"/>
+                    </oneOrMore>
+                </choice>
+            </element>
+        </optional>
+    </define>
+
+    <define name="element-last-fenced">
+        <element name="last-fenced">
+            <attribute name="target"> <text /> </attribute>
+            <attribute name="when"> <text /> </attribute>
+        </element>
+    </define>
+
+    <define name="element-validation">
+        <element name="validate">
+            <attribute name="agent"> <text /> </attribute>
+            <attribute name="valid"> <data type="boolean" /> </attribute>
+            <optional>
+                <externalRef href="command-output-1.0.rng" />
+            </optional>
+        </element>
+    </define>
+</grammar>
-- 
2.31.1

From 526a4148ba548a3dfec4394c9d10a8d71d18b81e Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Sun, 4 Sep 2022 01:15:59 -0700
Subject: [PATCH 13/22] Fix: schemas: crm_resource --validate validation fails

In case of an error, the output of `crm_resource --validate` fails to
validate (if validate-all does not output XML). This is because if a
<choice> contains two elements with the same name in a RelaxNG schema,
only the first occurrence is honored and the rest are ignored. (This
does not seem to be documented clearly; it's a conclusion based on
experimentation.)

The solution is to create just one <element> that contains a <choice>
(instead of a <choice> that contains two <element>s).

Closes RHBZ#2123727

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 xml/Makefile.am                    |  2 +-
 xml/api/command-output-2.23.rng    | 14 +-------------
 xml/api/crm_resource-2.23.rng      | 10 +++++-----
 xml/api/stonith_admin-2.23.rng     |  2 +-
 xml/api/subprocess-output-2.23.rng | 24 ++++++++++++++++++++++++
 5 files changed, 32 insertions(+), 20 deletions(-)
 create mode 100644 xml/api/subprocess-output-2.23.rng

diff --git a/xml/Makefile.am b/xml/Makefile.am
index 39f02f5..0a4a8aa 100644
--- a/xml/Makefile.am
+++ b/xml/Makefile.am
@@ -69,7 +69,7 @@ API_request_base	= command-output	\
 CIB_cfg_base		= options nodes resources constraints fencing acls tags alerts
 
 # Names of all schemas (including top level and those included by others)
-API_base		= $(API_request_base) fence-event failure generic-list item node-attrs node-history nodes resources status
+API_base		= $(API_request_base) fence-event failure generic-list item node-attrs node-history nodes resources status subprocess-output
 CIB_base		= cib $(CIB_cfg_base) status score rule nvset
 
 # Static schema files and transforms (only CIB has transforms)
diff --git a/xml/api/command-output-2.23.rng b/xml/api/command-output-2.23.rng
index 710c134..4de49bd 100644
--- a/xml/api/command-output-2.23.rng
+++ b/xml/api/command-output-2.23.rng
@@ -8,19 +8,7 @@
 
     <define name="command-output">
         <element name="command">
-            <attribute name="code"> <data type="integer" /> </attribute>
-            <optional>
-                <element name="output">
-                    <attribute name="source"><value>stdout</value></attribute>
-                    <text />
-                </element>
-            </optional>
-            <optional>
-                <element name="output">
-                    <attribute name="source"><value>stderr</value></attribute>
-                    <text />
-                </element>
-            </optional>
+            <externalRef href="subprocess-output-2.23.rng" />
         </element>
     </define>
 </grammar>
diff --git a/xml/api/crm_resource-2.23.rng b/xml/api/crm_resource-2.23.rng
index 8a46675..f841026 100644
--- a/xml/api/crm_resource-2.23.rng
+++ b/xml/api/crm_resource-2.23.rng
@@ -229,12 +229,12 @@
             </optional>
             <ref name="agent-status"/>
             <optional>
-                <choice>
-                    <element name="command">
+                <element name="command">
+                    <choice>
                         <text />
-                    </element>
-                    <externalRef href="command-output-1.0.rng"/>
-                </choice>
+                        <externalRef href="subprocess-output-2.23.rng"/>
+                    </choice>
+                </element>
             </optional>
         </element>
     </define>
diff --git a/xml/api/stonith_admin-2.23.rng b/xml/api/stonith_admin-2.23.rng
index b55fae9..f3fab68 100644
--- a/xml/api/stonith_admin-2.23.rng
+++ b/xml/api/stonith_admin-2.23.rng
@@ -45,7 +45,7 @@
             <attribute name="agent"> <text /> </attribute>
             <attribute name="valid"> <data type="boolean" /> </attribute>
             <optional>
-                <externalRef href="command-output-1.0.rng" />
+                <externalRef href="command-output-2.23.rng" />
             </optional>
         </element>
     </define>
diff --git a/xml/api/subprocess-output-2.23.rng b/xml/api/subprocess-output-2.23.rng
new file mode 100644
index 0000000..2f7a8e7
--- /dev/null
+++ b/xml/api/subprocess-output-2.23.rng
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="subprocess-output" />
+    </start>
+
+    <define name="subprocess-output">
+        <attribute name="code"> <data type="integer" /> </attribute>
+        <optional>
+            <element name="output">
+                <attribute name="source"><value>stdout</value></attribute>
+                <text />
+            </element>
+        </optional>
+        <optional>
+            <element name="output">
+                <attribute name="source"><value>stderr</value></attribute>
+                <text />
+            </element>
+        </optional>
+    </define>
+</grammar>
-- 
2.31.1

From 60af39cd1582bcf91ebcfc5f9ce2fc98fd14b5b9 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Sat, 27 Aug 2022 22:46:38 -0700
Subject: [PATCH 14/22] Low: schemas: Add schema for crm_error

This matches the current capabilities of crm_error, though we might want
to change to a oneOrMore choice for name and description later.

Closes T97

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 xml/api/crm_error-2.23.rng | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 xml/api/crm_error-2.23.rng

diff --git a/xml/api/crm_error-2.23.rng b/xml/api/crm_error-2.23.rng
new file mode 100644
index 0000000..8ba6e62
--- /dev/null
+++ b/xml/api/crm_error-2.23.rng
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-crm_error" />
+    </start>
+
+    <define name="element-crm_error">
+        <zeroOrMore>
+            <ref name="element-result-code" />
+        </zeroOrMore>
+    </define>
+
+    <define name="element-result-code">
+        <element name="result-code">
+            <attribute name="code"> <data type="integer" /> </attribute>
+            <attribute name="description"> <text /> </attribute>
+            <optional>
+                <attribute name="name"> <text /> </attribute>
+            </optional>
+        </element>
+    </define>
+</grammar>
-- 
2.31.1

From 4dbb0e9d79dd36647fbb222bd5c2adae518e541c Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Wed, 14 Sep 2022 22:53:49 -0700
Subject: [PATCH 15/22] Low: schemas: Copy API schemas in preparation for
 changes

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 include/crm/common/output_internal.h |   2 +-
 xml/api/crm_mon-2.24.rng             | 186 +++++++++++++++
 xml/api/crm_resource-2.24.rng        | 288 +++++++++++++++++++++++
 xml/api/crm_simulate-2.24.rng        | 338 +++++++++++++++++++++++++++
 xml/api/nodes-2.24.rng               |  54 +++++
 xml/api/resources-2.24.rng           | 109 +++++++++
 6 files changed, 976 insertions(+), 1 deletion(-)
 create mode 100644 xml/api/crm_mon-2.24.rng
 create mode 100644 xml/api/crm_resource-2.24.rng
 create mode 100644 xml/api/crm_simulate-2.24.rng
 create mode 100644 xml/api/nodes-2.24.rng
 create mode 100644 xml/api/resources-2.24.rng

diff --git a/include/crm/common/output_internal.h b/include/crm/common/output_internal.h
index 1e71e13..6c6d5a3 100644
--- a/include/crm/common/output_internal.h
+++ b/include/crm/common/output_internal.h
@@ -28,7 +28,7 @@ extern "C" {
  */
 
 
-#  define PCMK__API_VERSION "2.23"
+#  define PCMK__API_VERSION "2.24"
 
 #if defined(PCMK__WITH_ATTRIBUTE_OUTPUT_ARGS)
 #  define PCMK__OUTPUT_ARGS(ARGS...) __attribute__((output_args(ARGS)))
diff --git a/xml/api/crm_mon-2.24.rng b/xml/api/crm_mon-2.24.rng
new file mode 100644
index 0000000..b52307a
--- /dev/null
+++ b/xml/api/crm_mon-2.24.rng
@@ -0,0 +1,186 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-crm-mon"/>
+    </start>
+
+    <define name="element-crm-mon">
+        <optional>
+            <ref name="element-summary" />
+        </optional>
+        <optional>
+            <ref name="nodes-list" />
+        </optional>
+        <optional>
+            <ref name="resources-list" />
+        </optional>
+        <optional>
+            <ref name="node-attributes-list" />
+        </optional>
+        <optional>
+            <externalRef href="node-history-2.12.rng"/>
+        </optional>
+        <optional>
+            <ref name="failures-list" />
+        </optional>
+        <optional>
+            <ref name="fence-event-list" />
+        </optional>
+        <optional>
+            <ref name="tickets-list" />
+        </optional>
+        <optional>
+            <ref name="bans-list" />
+        </optional>
+    </define>
+
+    <define name="element-summary">
+        <element name="summary">
+            <optional>
+                <element name="stack">
+                    <attribute name="type"> <text /> </attribute>
+                </element>
+            </optional>
+            <optional>
+                <element name="current_dc">
+                    <attribute name="present"> <data type="boolean" /> </attribute>
+                    <optional>
+                        <group>
+                            <attribute name="version"> <text /> </attribute>
+                            <attribute name="name"> <text /> </attribute>
+                            <attribute name="id"> <text /> </attribute>
+                            <attribute name="with_quorum"> <data type="boolean" /> </attribute>
+                        </group>
+                    </optional>
+                    <optional>
+                        <attribute name="mixed_version"> <data type="boolean" /> </attribute>
+                    </optional>
+                </element>
+            </optional>
+            <optional>
+                <element name="last_update">
+                    <attribute name="time"> <text /> </attribute>
+                </element>
+                <element name="last_change">
+                    <attribute name="time"> <text /> </attribute>
+                    <attribute name="user"> <text /> </attribute>
+                    <attribute name="client"> <text /> </attribute>
+                    <attribute name="origin"> <text /> </attribute>
+                </element>
+            </optional>
+            <optional>
+                <element name="nodes_configured">
+                    <attribute name="number"> <data type="nonNegativeInteger" /> </attribute>
+                </element>
+                <element name="resources_configured">
+                    <attribute name="number"> <data type="nonNegativeInteger" /> </attribute>
+                    <attribute name="disabled"> <data type="nonNegativeInteger" /> </attribute>
+                    <attribute name="blocked"> <data type="nonNegativeInteger" /> </attribute>
+                </element>
+            </optional>
+            <optional>
+                <element name="cluster_options">
+                    <attribute name="stonith-enabled"> <data type="boolean" /> </attribute>
+                    <attribute name="symmetric-cluster"> <data type="boolean" /> </attribute>
+                    <attribute name="no-quorum-policy"> <text /> </attribute>
+                    <attribute name="maintenance-mode"> <data type="boolean" /> </attribute>
+                    <attribute name="stop-all-resources"> <data type="boolean" /> </attribute>
+                    <attribute name="stonith-timeout-ms"> <data type="integer" /> </attribute>
+                    <attribute name="priority-fencing-delay-ms"> <data type="integer" /> </attribute>
+                </element>
+            </optional>
+        </element>
+    </define>
+
+    <define name="resources-list">
+        <element name="resources">
+            <zeroOrMore>
+                <externalRef href="resources-2.24.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="nodes-list">
+        <element name="nodes">
+            <zeroOrMore>
+                <externalRef href="nodes-2.24.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="node-attributes-list">
+        <element name="node_attributes">
+            <zeroOrMore>
+                <externalRef href="node-attrs-2.8.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="failures-list">
+        <element name="failures">
+            <zeroOrMore>
+                <externalRef href="failure-2.8.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="fence-event-list">
+        <element name="fence_history">
+            <optional>
+                <attribute name="status"> <data type="integer" /> </attribute>
+            </optional>
+            <zeroOrMore>
+                <externalRef href="fence-event-2.15.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="tickets-list">
+        <element name="tickets">
+            <zeroOrMore>
+                <ref name="element-ticket" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="bans-list">
+        <element name="bans">
+            <zeroOrMore>
+                <ref name="element-ban" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="element-ticket">
+        <element name="ticket">
+            <attribute name="id"> <text /> </attribute>
+            <attribute name="status">
+                <choice>
+                    <value>granted</value>
+                    <value>revoked</value>
+                </choice>
+            </attribute>
+            <attribute name="standby"> <data type="boolean" /> </attribute>
+            <optional>
+                <attribute name="last-granted"> <text /> </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="element-ban">
+        <element name="ban">
+            <attribute name="id"> <text /> </attribute>
+            <attribute name="resource"> <text /> </attribute>
+            <attribute name="node"> <text /> </attribute>
+            <attribute name="weight"> <data type="integer" /> </attribute>
+            <attribute name="promoted-only"> <data type="boolean" /> </attribute>
+            <!-- DEPRECATED: master_only is a duplicate of promoted-only that is
+                 provided solely for API backward compatibility. It will be
+                 removed in a future release. Check promoted-only instead.
+              -->
+            <attribute name="master_only"> <data type="boolean" /> </attribute>
+        </element>
+    </define>
+</grammar>
diff --git a/xml/api/crm_resource-2.24.rng b/xml/api/crm_resource-2.24.rng
new file mode 100644
index 0000000..6a3334c
--- /dev/null
+++ b/xml/api/crm_resource-2.24.rng
@@ -0,0 +1,288 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-crm-resource"/>
+    </start>
+
+    <define name="element-crm-resource">
+        <choice>
+            <ref name="agents-list" />
+            <ref name="alternatives-list" />
+            <ref name="constraints-list" />
+            <externalRef href="generic-list-2.4.rng"/>
+            <element name="metadata"> <text/> </element>
+            <ref name="locate-list" />
+            <ref name="operations-list" />
+            <ref name="providers-list" />
+            <ref name="reasons-list" />
+            <ref name="resource-check" />
+            <ref name="resource-config" />
+            <ref name="resources-list" />
+            <ref name="resource-agent-action" />
+        </choice>
+    </define>
+
+    <define name="agents-list">
+        <element name="agents">
+            <attribute name="standard"> <text/> </attribute>
+            <optional>
+                <attribute name="provider"> <text/> </attribute>
+            </optional>
+            <zeroOrMore>
+                <element name="agent"> <text/> </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="alternatives-list">
+        <element name="providers">
+            <attribute name="for"> <text/> </attribute>
+            <zeroOrMore>
+                <element name="provider"> <text/> </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="constraints-list">
+        <element name="constraints">
+            <interleave>
+                <zeroOrMore>
+                    <ref name="rsc-location" />
+                </zeroOrMore>
+                <zeroOrMore>
+                    <ref name="rsc-colocation" />
+                </zeroOrMore>
+            </interleave>
+        </element>
+    </define>
+
+    <define name="locate-list">
+        <element name="nodes">
+            <attribute name="resource"> <text/> </attribute>
+            <zeroOrMore>
+                <element name="node">
+                    <optional>
+                        <attribute name="state"><value>promoted</value></attribute>
+                    </optional>
+                    <text/>
+                </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="rsc-location">
+        <element name="rsc_location">
+            <attribute name="node"> <text/> </attribute>
+            <attribute name="rsc"> <text/> </attribute>
+            <attribute name="id"> <text/> </attribute>
+            <externalRef href="../score.rng"/>
+        </element>
+    </define>
+
+    <define name="operations-list">
+        <element name="operations">
+            <oneOrMore>
+                <ref name="element-operation-list" />
+            </oneOrMore>
+        </element>
+    </define>
+
+    <define name="providers-list">
+        <element name="providers">
+            <attribute name="standard"> <value>ocf</value> </attribute>
+            <optional>
+                <attribute name="agent"> <text/> </attribute>
+            </optional>
+            <zeroOrMore>
+                <element name="provider"> <text/> </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="reasons-list">
+        <element name="reason">
+            <!-- set only when resource and node are both specified -->
+            <optional>
+                <attribute name="running_on"> <text/> </attribute>
+            </optional>
+
+            <!-- set only when only a resource is specified -->
+            <optional>
+                <attribute name="running"> <data type="boolean"/> </attribute>
+            </optional>
+
+            <choice>
+                <ref name="reasons-with-no-resource"/>
+                <ref name="resource-check"/>
+            </choice>
+        </element>
+    </define>
+
+    <define name="reasons-with-no-resource">
+        <element name="resources">
+            <zeroOrMore>
+                <element name="resource">
+                    <attribute name="id"> <text/> </attribute>
+                    <attribute name="running"> <data type="boolean"/> </attribute>
+                    <optional>
+                        <attribute name="host"> <text/> </attribute>
+                    </optional>
+                    <ref name="resource-check"/>
+                </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="resource-config">
+        <element name="resource_config">
+            <externalRef href="resources-2.24.rng" />
+            <element name="xml"> <text/> </element>
+        </element>
+    </define>
+
+    <define name="resource-check">
+        <element name="check">
+            <attribute name="id"> <text/> </attribute>
+            <optional>
+                <choice>
+                    <attribute name="remain_stopped"><value>true</value></attribute>
+                    <attribute name="promotable"><value>false</value></attribute>
+                </choice>
+            </optional>
+            <optional>
+                <attribute name="unmanaged"><value>true</value></attribute>
+            </optional>
+            <optional>
+                <attribute name="locked-to"> <text/> </attribute>
+            </optional>
+            <optional>
+                <attribute name="unhealthy"><value>true</value></attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="resources-list">
+        <element name="resources">
+            <zeroOrMore>
+                <externalRef href="resources-2.24.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="rsc-colocation">
+        <element name="rsc_colocation">
+            <attribute name="id"> <text/> </attribute>
+            <attribute name="rsc"> <text/> </attribute>
+            <attribute name="with-rsc"> <text/> </attribute>
+            <externalRef href="../score.rng"/>
+            <optional>
+                <attribute name="node-attribute"> <text/> </attribute>
+            </optional>
+            <optional>
+                <attribute name="rsc-role">
+                    <ref name="attribute-roles"/>
+                </attribute>
+            </optional>
+            <optional>
+                <attribute name="with-rsc-role">
+                    <ref name="attribute-roles"/>
+                </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="element-operation-list">
+        <element name="operation">
+            <optional>
+                <group>
+                    <attribute name="rsc"> <text/> </attribute>
+                    <attribute name="agent"> <text/> </attribute>
+                </group>
+            </optional>
+            <attribute name="op"> <text/> </attribute>
+            <attribute name="node"> <text/> </attribute>
+            <attribute name="call"> <data type="integer" /> </attribute>
+            <attribute name="rc"> <data type="nonNegativeInteger" /> </attribute>
+            <optional>
+                <attribute name="last-rc-change"> <text/> </attribute>
+                <attribute name="exec-time"> <data type="nonNegativeInteger" /> </attribute>
+            </optional>
+            <attribute name="status"> <text/> </attribute>
+        </element>
+    </define>
+
+    <define name="resource-agent-action">
+        <element name="resource-agent-action">
+            <attribute name="action"> <text/> </attribute>
+            <optional>
+                <attribute name="rsc"> <text/> </attribute>
+            </optional>
+            <attribute name="class"> <text/> </attribute>
+            <attribute name="type"> <text/> </attribute>
+            <optional>
+                <attribute name="provider"> <text/> </attribute>
+            </optional>
+            <optional>
+                <ref name="overrides-list"/>
+            </optional>
+            <ref name="agent-status"/>
+            <optional>
+                <element name="command">
+                    <choice>
+                        <text />
+                        <externalRef href="subprocess-output-2.23.rng"/>
+                    </choice>
+                </element>
+            </optional>
+        </element>
+    </define>
+
+    <define name="overrides-list">
+        <element name="overrides">
+            <zeroOrMore>
+                <element name="override">
+                    <optional>
+                        <attribute name="rsc"> <text/> </attribute>
+                    </optional>
+                    <attribute name="name"> <text/> </attribute>
+                    <attribute name="value"> <text/> </attribute>
+                </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="agent-status">
+        <element name="agent-status">
+            <attribute name="code"> <data type="integer" /> </attribute>
+            <optional>
+                <attribute name="message"> <text/> </attribute>
+            </optional>
+            <optional>
+                <attribute name="execution_code"> <data type="integer" /> </attribute>
+            </optional>
+            <optional>
+                <attribute name="execution_message"> <text/> </attribute>
+            </optional>
+            <optional>
+                <attribute name="reason"> <text/> </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="attribute-roles">
+        <choice>
+            <value>Stopped</value>
+            <value>Started</value>
+            <value>Promoted</value>
+            <value>Unpromoted</value>
+
+            <!-- These synonyms for Promoted/Unpromoted are allowed for
+                 backward compatibility with output from older Pacemaker
+                 versions that used them -->
+            <value>Master</value>
+            <value>Slave</value>
+        </choice>
+    </define>
+</grammar>
diff --git a/xml/api/crm_simulate-2.24.rng b/xml/api/crm_simulate-2.24.rng
new file mode 100644
index 0000000..5be0afa
--- /dev/null
+++ b/xml/api/crm_simulate-2.24.rng
@@ -0,0 +1,338 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-crm-simulate"/>
+    </start>
+
+    <define name="element-crm-simulate">
+        <choice>
+            <ref name="timings-list" />
+            <group>
+                <ref name="cluster-status" />
+                <optional>
+                    <ref name="modifications-list" />
+                </optional>
+                <optional>
+                    <ref name="allocations-utilizations-list" />
+                </optional>
+                <optional>
+                    <ref name="action-list" />
+                </optional>
+                <optional>
+                    <ref name="cluster-injected-actions-list" />
+                    <ref name="revised-cluster-status" />
+                </optional>
+            </group>
+        </choice>
+    </define>
+
+    <define name="allocations-utilizations-list">
+        <choice>
+            <element name="allocations">
+                <zeroOrMore>
+                    <choice>
+                        <ref name="element-allocation" />
+                        <ref name="element-promotion" />
+                    </choice>
+                </zeroOrMore>
+            </element>
+            <element name="utilizations">
+                <zeroOrMore>
+                    <choice>
+                        <ref name="element-capacity" />
+                        <ref name="element-utilization" />
+                    </choice>
+                </zeroOrMore>
+            </element>
+            <element name="allocations_utilizations">
+                <zeroOrMore>
+                    <choice>
+                        <ref name="element-allocation" />
+                        <ref name="element-promotion" />
+                        <ref name="element-capacity" />
+                        <ref name="element-utilization" />
+                    </choice>
+                </zeroOrMore>
+            </element>
+        </choice>
+    </define>
+
+    <define name="cluster-status">
+        <element name="cluster_status">
+            <ref name="nodes-list" />
+            <ref name="resources-list" />
+            <optional>
+                <ref name="node-attributes-list" />
+            </optional>
+            <optional>
+                <externalRef href="node-history-2.12.rng" />
+            </optional>
+            <optional>
+                <ref name="failures-list" />
+            </optional>
+        </element>
+    </define>
+
+    <define name="modifications-list">
+        <element name="modifications">
+            <optional>
+                <attribute name="quorum"> <text /> </attribute>
+            </optional>
+            <optional>
+                <attribute name="watchdog"> <text /> </attribute>
+            </optional>
+            <zeroOrMore>
+                <ref name="element-inject-modify-node" />
+            </zeroOrMore>
+            <zeroOrMore>
+                <ref name="element-inject-modify-ticket" />
+            </zeroOrMore>
+            <zeroOrMore>
+                <ref name="element-inject-spec" />
+            </zeroOrMore>
+            <zeroOrMore>
+                <ref name="element-inject-attr" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="revised-cluster-status">
+        <element name="revised_cluster_status">
+            <ref name="nodes-list" />
+            <ref name="resources-list" />
+            <optional>
+                <ref name="node-attributes-list" />
+            </optional>
+            <optional>
+                <ref name="failures-list" />
+            </optional>
+        </element>
+    </define>
+
+    <define name="element-inject-attr">
+        <element name="inject_attr">
+            <attribute name="cib_node"> <text /> </attribute>
+            <attribute name="name"> <text /> </attribute>
+            <attribute name="node_path"> <text /> </attribute>
+            <attribute name="value"> <text /> </attribute>
+        </element>
+    </define>
+
+    <define name="element-inject-modify-node">
+        <element name="modify_node">
+            <attribute name="action"> <text /> </attribute>
+            <attribute name="node"> <text /> </attribute>
+        </element>
+    </define>
+
+    <define name="element-inject-spec">
+        <element name="inject_spec">
+            <attribute name="spec"> <text /> </attribute>
+        </element>
+    </define>
+
+    <define name="element-inject-modify-ticket">
+        <element name="modify_ticket">
+            <attribute name="action"> <text /> </attribute>
+            <attribute name="ticket"> <text /> </attribute>
+        </element>
+    </define>
+
+    <define name="cluster-injected-actions-list">
+        <element name="transition">
+            <zeroOrMore>
+                <ref name="element-injected-actions" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="node-attributes-list">
+        <element name="node_attributes">
+            <zeroOrMore>
+                <externalRef href="node-attrs-2.8.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="failures-list">
+        <element name="failures">
+            <zeroOrMore>
+                <externalRef href="failure-2.8.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="nodes-list">
+        <element name="nodes">
+            <zeroOrMore>
+                <externalRef href="nodes-2.24.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="resources-list">
+        <element name="resources">
+            <zeroOrMore>
+                <externalRef href="resources-2.24.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="timings-list">
+        <element name="timings">
+            <zeroOrMore>
+                <ref name="element-timing" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="action-list">
+        <element name="actions">
+            <zeroOrMore>
+                <ref name="element-node-action" />
+            </zeroOrMore>
+            <zeroOrMore>
+                <ref name="element-rsc-action" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="element-allocation">
+        <element name="node_weight">
+            <attribute name="function"> <text /> </attribute>
+            <attribute name="node"> <text /> </attribute>
+            <externalRef href="../score.rng" />
+            <optional>
+                <attribute name="id"> <text /> </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="element-capacity">
+        <element name="capacity">
+            <attribute name="comment"> <text /> </attribute>
+            <attribute name="node"> <text /> </attribute>
+            <zeroOrMore>
+                <element>
+                    <anyName />
+                    <text />
+                </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="element-inject-cluster-action">
+        <element name="cluster_action">
+            <attribute name="node"> <text /> </attribute>
+            <attribute name="task"> <text /> </attribute>
+            <optional>
+                <attribute name="id"> <text /> </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="element-injected-actions">
+        <choice>
+            <ref name="element-inject-cluster-action" />
+            <ref name="element-inject-fencing-action" />
+            <ref name="element-inject-pseudo-action" />
+            <ref name="element-inject-rsc-action" />
+        </choice>
+    </define>
+
+    <define name="element-inject-fencing-action">
+        <element name="fencing_action">
+            <attribute name="op"> <text /> </attribute>
+            <attribute name="target"> <text /> </attribute>
+        </element>
+    </define>
+
+    <define name="element-node-action">
+        <element name="node_action">
+            <attribute name="node"> <text /> </attribute>
+            <attribute name="reason"> <text /> </attribute>
+            <attribute name="task"> <text /> </attribute>
+        </element>
+    </define>
+
+    <define name="element-promotion">
+        <element name="promotion_score">
+            <attribute name="id"> <text /> </attribute>
+            <externalRef href="../score.rng" />
+            <optional>
+                <attribute name="node"> <text /> </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="element-inject-pseudo-action">
+        <element name="pseudo_action">
+            <attribute name="task"> <text /> </attribute>
+            <optional>
+                <attribute name="node"> <text /> </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="element-inject-rsc-action">
+        <element name="rsc_action">
+            <attribute name="node"> <text /> </attribute>
+            <attribute name="op"> <text /> </attribute>
+            <attribute name="resource"> <text /> </attribute>
+            <optional>
+                <attribute name="interval"> <data type="integer" /> </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="element-timing">
+        <element name="timing">
+            <attribute name="file"> <text /> </attribute>
+            <attribute name="duration"> <data type="double" /> </attribute>
+        </element>
+    </define>
+
+    <define name="element-rsc-action">
+        <element name="rsc_action">
+            <attribute name="action"> <text /> </attribute>
+            <attribute name="resource"> <text /> </attribute>
+            <optional>
+                <attribute name="blocked"> <data type="boolean" /> </attribute>
+            </optional>
+            <optional>
+                <attribute name="dest"> <text /> </attribute>
+            </optional>
+            <optional>
+                <attribute name="next-role"> <text /> </attribute>
+            </optional>
+            <optional>
+                <attribute name="node"> <text /> </attribute>
+            </optional>
+            <optional>
+                <attribute name="reason"> <text /> </attribute>
+            </optional>
+            <optional>
+                <attribute name="role"> <text /> </attribute>
+            </optional>
+            <optional>
+                <attribute name="source"> <text /> </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="element-utilization">
+        <element name="utilization">
+            <attribute name="function"> <text /> </attribute>
+            <attribute name="node"> <text /> </attribute>
+            <attribute name="resource"> <text /> </attribute>
+            <zeroOrMore>
+                <element>
+                    <anyName />
+                    <text />
+                </element>
+            </zeroOrMore>
+        </element>
+    </define>
+</grammar>
diff --git a/xml/api/nodes-2.24.rng b/xml/api/nodes-2.24.rng
new file mode 100644
index 0000000..9686344
--- /dev/null
+++ b/xml/api/nodes-2.24.rng
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-full-node"/>
+    </start>
+
+    <define name="element-full-node">
+        <element name="node">
+            <attribute name="name"> <text/> </attribute>
+            <attribute name="id"> <text/> </attribute>
+            <attribute name="online"> <data type="boolean" /> </attribute>
+            <attribute name="standby"> <data type="boolean" /> </attribute>
+            <attribute name="standby_onfail"> <data type="boolean" /> </attribute>
+            <attribute name="maintenance"> <data type="boolean" /> </attribute>
+            <attribute name="pending"> <data type="boolean" /> </attribute>
+            <attribute name="unclean"> <data type="boolean" /> </attribute>
+            <optional>
+                <attribute name="health">
+                    <choice>
+                        <value>red</value>
+                        <value>yellow</value>
+                        <value>green</value>
+                    </choice>
+                </attribute>
+            </optional>
+            <optional>
+                <attribute name="feature_set"> <text/> </attribute>
+            </optional>
+            <attribute name="shutdown"> <data type="boolean" /> </attribute>
+            <attribute name="expected_up"> <data type="boolean" /> </attribute>
+            <attribute name="is_dc"> <data type="boolean" /> </attribute>
+            <attribute name="resources_running"> <data type="nonNegativeInteger" /> </attribute>
+            <attribute name="type">
+                <choice>
+                    <value>unknown</value>
+                    <value>member</value>
+                    <value>remote</value>
+                    <value>ping</value>
+                </choice>
+            </attribute>
+            <optional>
+                <!-- for virtualized pacemaker_remote nodes, crm_mon 1.1.13 uses
+                     "container_id" while later versions use "id_as_resource" -->
+                <choice>
+                    <attribute name="container_id"> <text/> </attribute>
+                    <attribute name="id_as_resource"> <text/> </attribute>
+                </choice>
+            </optional>
+            <externalRef href="resources-2.24.rng" />
+        </element>
+    </define>
+</grammar>
diff --git a/xml/api/resources-2.24.rng b/xml/api/resources-2.24.rng
new file mode 100644
index 0000000..e279583
--- /dev/null
+++ b/xml/api/resources-2.24.rng
@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-resource-list"/>
+    </start>
+
+    <define name="element-resource-list">
+        <interleave>
+            <zeroOrMore>
+                <ref name="element-bundle" />
+            </zeroOrMore>
+            <zeroOrMore>
+                <ref name="element-clone" />
+            </zeroOrMore>
+            <zeroOrMore>
+                <ref name="element-group" />
+            </zeroOrMore>
+            <zeroOrMore>
+                <ref name="element-resource" />
+            </zeroOrMore>
+        </interleave>
+    </define>
+
+    <define name="element-bundle">
+        <element name="bundle">
+            <attribute name="id"> <text/> </attribute>
+            <attribute name="type">
+                <choice>
+                    <value>docker</value>
+                    <value>rkt</value>
+                    <value>podman</value>
+                </choice>
+            </attribute>
+            <attribute name="image"> <text/> </attribute>
+            <attribute name="unique"> <data type="boolean" /> </attribute>
+            <attribute name="managed"> <data type="boolean" /> </attribute>
+            <attribute name="failed"> <data type="boolean" /> </attribute>
+            <zeroOrMore>
+                <element name="replica">
+                    <attribute name="id"> <data type="nonNegativeInteger" /> </attribute>
+                    <zeroOrMore>
+                        <ref name="element-resource" />
+                    </zeroOrMore>
+                </element>
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="element-clone">
+        <element name="clone">
+            <attribute name="id"> <text/> </attribute>
+            <attribute name="multi_state"> <data type="boolean" /> </attribute>
+            <attribute name="unique"> <data type="boolean" /> </attribute>
+            <attribute name="managed"> <data type="boolean" /> </attribute>
+            <attribute name="disabled"> <data type="boolean" /> </attribute>
+            <attribute name="failed"> <data type="boolean" /> </attribute>
+            <attribute name="failure_ignored"> <data type="boolean" /> </attribute>
+            <optional>
+                <attribute name="target_role"> <text/> </attribute>
+            </optional>
+            <ref name="element-resource-list" />
+        </element>
+    </define>
+
+    <define name="element-group">
+        <element name="group">
+            <attribute name="id"> <text/> </attribute>
+            <attribute name="number_resources"> <data type="nonNegativeInteger" /> </attribute>
+            <attribute name="managed"> <data type="boolean" /> </attribute>
+            <attribute name="disabled"> <data type="boolean" /> </attribute>
+            <ref name="element-resource-list" />
+        </element>
+    </define>
+
+    <define name="element-resource">
+        <element name="resource">
+            <attribute name="id"> <text/> </attribute>
+            <attribute name="resource_agent"> <text/> </attribute>
+            <attribute name="role"> <text/> </attribute>
+            <optional>
+                <attribute name="target_role"> <text/> </attribute>
+            </optional>
+            <attribute name="active"> <data type="boolean" /> </attribute>
+            <attribute name="orphaned"> <data type="boolean" /> </attribute>
+            <optional>
+                <attribute name="blocked"> <data type="boolean" /> </attribute>
+            </optional>
+            <attribute name="managed"> <data type="boolean" /> </attribute>
+            <attribute name="failed"> <data type="boolean" /> </attribute>
+            <attribute name="failure_ignored"> <data type="boolean" /> </attribute>
+            <attribute name="nodes_running_on"> <data type="nonNegativeInteger" />  </attribute>
+            <optional>
+                <attribute name="pending"> <text/> </attribute>
+            </optional>
+            <zeroOrMore>
+                <element name="node">
+                    <attribute name="name"> <text/> </attribute>
+                    <attribute name="id"> <text/> </attribute>
+                    <attribute name="cached"> <data type="boolean" /> </attribute>
+                </element>
+            </zeroOrMore>
+            <optional>
+                <element name="xml"> <text/> </element>
+            </optional>
+        </element>
+    </define>
+</grammar>
-- 
2.31.1

From 1d36b5d50e071ecaa66948066f23043a513871e8 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Wed, 14 Sep 2022 22:58:04 -0700
Subject: [PATCH 16/22] API: schemas: Add locked_to= to resources API schema

Ref T433

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 xml/api/resources-2.24.rng | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/xml/api/resources-2.24.rng b/xml/api/resources-2.24.rng
index e279583..f8ae6eb 100644
--- a/xml/api/resources-2.24.rng
+++ b/xml/api/resources-2.24.rng
@@ -94,6 +94,9 @@
             <optional>
                 <attribute name="pending"> <text/> </attribute>
             </optional>
+            <optional>
+                <attribute name="locked_to"> <text/> </attribute>
+            </optional>
             <zeroOrMore>
                 <element name="node">
                     <attribute name="name"> <text/> </attribute>
-- 
2.31.1

From e8caa027408243a6c7edfa966a1a7b0535458b9a Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 21:53:02 -0700
Subject: [PATCH 17/22] Low: schemas: Copy API schemas in preparation for
 changes

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 include/crm/common/output_internal.h |   2 +-
 xml/api/crm_mon-2.25.rng             | 186 +++++++++++++++++++++++++++
 xml/api/crmadmin-2.25.rng            |  68 ++++++++++
 3 files changed, 255 insertions(+), 1 deletion(-)
 create mode 100644 xml/api/crm_mon-2.25.rng
 create mode 100644 xml/api/crmadmin-2.25.rng

diff --git a/include/crm/common/output_internal.h b/include/crm/common/output_internal.h
index 6c6d5a3..1974721 100644
--- a/include/crm/common/output_internal.h
+++ b/include/crm/common/output_internal.h
@@ -28,7 +28,7 @@ extern "C" {
  */
 
 
-#  define PCMK__API_VERSION "2.24"
+#  define PCMK__API_VERSION "2.25"
 
 #if defined(PCMK__WITH_ATTRIBUTE_OUTPUT_ARGS)
 #  define PCMK__OUTPUT_ARGS(ARGS...) __attribute__((output_args(ARGS)))
diff --git a/xml/api/crm_mon-2.25.rng b/xml/api/crm_mon-2.25.rng
new file mode 100644
index 0000000..b52307a
--- /dev/null
+++ b/xml/api/crm_mon-2.25.rng
@@ -0,0 +1,186 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-crm-mon"/>
+    </start>
+
+    <define name="element-crm-mon">
+        <optional>
+            <ref name="element-summary" />
+        </optional>
+        <optional>
+            <ref name="nodes-list" />
+        </optional>
+        <optional>
+            <ref name="resources-list" />
+        </optional>
+        <optional>
+            <ref name="node-attributes-list" />
+        </optional>
+        <optional>
+            <externalRef href="node-history-2.12.rng"/>
+        </optional>
+        <optional>
+            <ref name="failures-list" />
+        </optional>
+        <optional>
+            <ref name="fence-event-list" />
+        </optional>
+        <optional>
+            <ref name="tickets-list" />
+        </optional>
+        <optional>
+            <ref name="bans-list" />
+        </optional>
+    </define>
+
+    <define name="element-summary">
+        <element name="summary">
+            <optional>
+                <element name="stack">
+                    <attribute name="type"> <text /> </attribute>
+                </element>
+            </optional>
+            <optional>
+                <element name="current_dc">
+                    <attribute name="present"> <data type="boolean" /> </attribute>
+                    <optional>
+                        <group>
+                            <attribute name="version"> <text /> </attribute>
+                            <attribute name="name"> <text /> </attribute>
+                            <attribute name="id"> <text /> </attribute>
+                            <attribute name="with_quorum"> <data type="boolean" /> </attribute>
+                        </group>
+                    </optional>
+                    <optional>
+                        <attribute name="mixed_version"> <data type="boolean" /> </attribute>
+                    </optional>
+                </element>
+            </optional>
+            <optional>
+                <element name="last_update">
+                    <attribute name="time"> <text /> </attribute>
+                </element>
+                <element name="last_change">
+                    <attribute name="time"> <text /> </attribute>
+                    <attribute name="user"> <text /> </attribute>
+                    <attribute name="client"> <text /> </attribute>
+                    <attribute name="origin"> <text /> </attribute>
+                </element>
+            </optional>
+            <optional>
+                <element name="nodes_configured">
+                    <attribute name="number"> <data type="nonNegativeInteger" /> </attribute>
+                </element>
+                <element name="resources_configured">
+                    <attribute name="number"> <data type="nonNegativeInteger" /> </attribute>
+                    <attribute name="disabled"> <data type="nonNegativeInteger" /> </attribute>
+                    <attribute name="blocked"> <data type="nonNegativeInteger" /> </attribute>
+                </element>
+            </optional>
+            <optional>
+                <element name="cluster_options">
+                    <attribute name="stonith-enabled"> <data type="boolean" /> </attribute>
+                    <attribute name="symmetric-cluster"> <data type="boolean" /> </attribute>
+                    <attribute name="no-quorum-policy"> <text /> </attribute>
+                    <attribute name="maintenance-mode"> <data type="boolean" /> </attribute>
+                    <attribute name="stop-all-resources"> <data type="boolean" /> </attribute>
+                    <attribute name="stonith-timeout-ms"> <data type="integer" /> </attribute>
+                    <attribute name="priority-fencing-delay-ms"> <data type="integer" /> </attribute>
+                </element>
+            </optional>
+        </element>
+    </define>
+
+    <define name="resources-list">
+        <element name="resources">
+            <zeroOrMore>
+                <externalRef href="resources-2.24.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="nodes-list">
+        <element name="nodes">
+            <zeroOrMore>
+                <externalRef href="nodes-2.24.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="node-attributes-list">
+        <element name="node_attributes">
+            <zeroOrMore>
+                <externalRef href="node-attrs-2.8.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="failures-list">
+        <element name="failures">
+            <zeroOrMore>
+                <externalRef href="failure-2.8.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="fence-event-list">
+        <element name="fence_history">
+            <optional>
+                <attribute name="status"> <data type="integer" /> </attribute>
+            </optional>
+            <zeroOrMore>
+                <externalRef href="fence-event-2.15.rng" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="tickets-list">
+        <element name="tickets">
+            <zeroOrMore>
+                <ref name="element-ticket" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="bans-list">
+        <element name="bans">
+            <zeroOrMore>
+                <ref name="element-ban" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="element-ticket">
+        <element name="ticket">
+            <attribute name="id"> <text /> </attribute>
+            <attribute name="status">
+                <choice>
+                    <value>granted</value>
+                    <value>revoked</value>
+                </choice>
+            </attribute>
+            <attribute name="standby"> <data type="boolean" /> </attribute>
+            <optional>
+                <attribute name="last-granted"> <text /> </attribute>
+            </optional>
+        </element>
+    </define>
+
+    <define name="element-ban">
+        <element name="ban">
+            <attribute name="id"> <text /> </attribute>
+            <attribute name="resource"> <text /> </attribute>
+            <attribute name="node"> <text /> </attribute>
+            <attribute name="weight"> <data type="integer" /> </attribute>
+            <attribute name="promoted-only"> <data type="boolean" /> </attribute>
+            <!-- DEPRECATED: master_only is a duplicate of promoted-only that is
+                 provided solely for API backward compatibility. It will be
+                 removed in a future release. Check promoted-only instead.
+              -->
+            <attribute name="master_only"> <data type="boolean" /> </attribute>
+        </element>
+    </define>
+</grammar>
diff --git a/xml/api/crmadmin-2.25.rng b/xml/api/crmadmin-2.25.rng
new file mode 100644
index 0000000..34c9ca4
--- /dev/null
+++ b/xml/api/crmadmin-2.25.rng
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-crmadmin"/>
+    </start>
+
+    <define name="element-crmadmin">
+        <optional>
+            <ref name="element-status" />
+        </optional>
+        <optional>
+            <ref name="element-pacemakerd" />
+        </optional>
+        <optional>
+            <ref name="element-dc" />
+        </optional>
+        <optional>
+            <ref name="crmadmin-nodes-list" />
+        </optional>
+    </define>
+
+    <define name="element-status">
+        <element name="crmd">
+            <attribute name="node_name"> <text /> </attribute>
+            <attribute name="state"> <text /> </attribute>
+            <attribute name="result"> <text /> </attribute>
+        </element>
+    </define>
+
+    <define name="element-pacemakerd">
+        <element name="pacemakerd">
+            <attribute name="state"> <text /> </attribute>
+            <attribute name="last_updated"> <text /> </attribute>
+        </element>
+    </define>
+
+    <define name="element-dc">
+        <element name="dc">
+            <attribute name="node_name"> <text /> </attribute>
+        </element>
+    </define>
+
+    <define name="crmadmin-nodes-list">
+        <element name="nodes">
+            <zeroOrMore>
+                <ref name="element-crmadmin-node" />
+            </zeroOrMore>
+        </element>
+    </define>
+
+    <define name="element-crmadmin-node">
+        <element name="node">
+            <attribute name="type">
+                <choice>
+                    <value>unknown</value>
+                    <value>member</value>
+                    <value>remote</value>
+                    <value>ping</value>
+                </choice>
+            </attribute>
+
+            <attribute name="name"> <text/> </attribute>
+            <attribute name="id"> <text/> </attribute>
+        </element>
+    </define>
+</grammar>
-- 
2.31.1

From 9e06f1b526e9ceb94cc1709e245537d169ca2952 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 21:55:51 -0700
Subject: [PATCH 18/22] Low: schemas: Add pacemakerd-health schema in
 preparation for fix

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 xml/api/crm_mon-2.25.rng           |  3 +++
 xml/api/crmadmin-2.25.rng          |  9 +--------
 xml/api/pacemakerd-health-2.25.rng | 20 ++++++++++++++++++++
 3 files changed, 24 insertions(+), 8 deletions(-)
 create mode 100644 xml/api/pacemakerd-health-2.25.rng

diff --git a/xml/api/crm_mon-2.25.rng b/xml/api/crm_mon-2.25.rng
index b52307a..1e501dd 100644
--- a/xml/api/crm_mon-2.25.rng
+++ b/xml/api/crm_mon-2.25.rng
@@ -7,6 +7,9 @@
     </start>
 
     <define name="element-crm-mon">
+        <optional>
+            <externalRef href="pacemakerd-health-2.25.rng" />
+        </optional>
         <optional>
             <ref name="element-summary" />
         </optional>
diff --git a/xml/api/crmadmin-2.25.rng b/xml/api/crmadmin-2.25.rng
index 34c9ca4..973f6d4 100644
--- a/xml/api/crmadmin-2.25.rng
+++ b/xml/api/crmadmin-2.25.rng
@@ -11,7 +11,7 @@
             <ref name="element-status" />
         </optional>
         <optional>
-            <ref name="element-pacemakerd" />
+            <externalRef href="pacemakerd-health-2.25.rng" />
         </optional>
         <optional>
             <ref name="element-dc" />
@@ -29,13 +29,6 @@
         </element>
     </define>
 
-    <define name="element-pacemakerd">
-        <element name="pacemakerd">
-            <attribute name="state"> <text /> </attribute>
-            <attribute name="last_updated"> <text /> </attribute>
-        </element>
-    </define>
-
     <define name="element-dc">
         <element name="dc">
             <attribute name="node_name"> <text /> </attribute>
diff --git a/xml/api/pacemakerd-health-2.25.rng b/xml/api/pacemakerd-health-2.25.rng
new file mode 100644
index 0000000..2089b25
--- /dev/null
+++ b/xml/api/pacemakerd-health-2.25.rng
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0"
+         datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+
+    <start>
+        <ref name="element-pacemakerd-health"/>
+    </start>
+
+    <define name="element-pacemakerd-health">
+        <element name="pacemakerd">
+            <optional>
+                <attribute name="sys_from"> <text /> </attribute>
+            </optional>
+            <attribute name="state"> <text /> </attribute>
+            <optional>
+                <attribute name="last_updated"> <text /> </attribute>
+            </optional>
+        </element>
+    </define>
+</grammar>
-- 
2.31.1

From 9a320b51e21e4c52a5ac3332d35c0d70fdd1650c Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 22:03:19 -0700
Subject: [PATCH 19/22] Low: libpacemaker: Fix pacemakerd-health XML output

We were using F_CRM_SYS_FROM as the name of the XML element, instead of
something static like "pacemakerd". It happens that the value in
F_CRM_SYS_FROM seems to always be CRM_SYSTEM_MCP ("pacemakerd"), so the
element name was effectively deterministic. Nonetheless, the schema
required the element be called "pacemakerd"; there was no allowance for
another system name. That defeats any purpose of flexible element
naming.

It seems better to call the element "pacemakerd" and make
sys_from a field, if we keep sys_from at all. (Can't use
"pacemakerd-health" for backward compatibility reasons.)

Additionally, if sys_from or last_updated is NULL, pass them directly to
pcmk__output_create_xml_node(). Those attributes will simply be skipped
if their values are NULL.

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 lib/pacemaker/pcmk_output.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/pacemaker/pcmk_output.c b/lib/pacemaker/pcmk_output.c
index c088a6a..153a422 100644
--- a/lib/pacemaker/pcmk_output.c
+++ b/lib/pacemaker/pcmk_output.c
@@ -710,9 +710,10 @@ pacemakerd_health_xml(pcmk__output_t *out, va_list args)
         state_s = pcmk_pacemakerd_api_daemon_state_enum2text(state);
     }
 
-    pcmk__output_create_xml_node(out, crm_str(sys_from),
+    pcmk__output_create_xml_node(out, "pacemakerd",
+                                 "sys_from", sys_from,
                                  "state", state_s,
-                                 "last_updated", crm_str(last_updated),
+                                 "last_updated", last_updated,
                                  NULL);
     return pcmk_rc_ok;
 }
-- 
2.31.1

From bb57ee10fe6eaeaaeafbf8b491b446f7bffb6b22 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Tue, 11 Oct 2022 15:14:27 -0700
Subject: [PATCH 20/22] Refactor: libpacemaker: Default to sync dispatch in
 pcmk_cluster_queries

If message_timeout_ms == 0 for various functions in
pcmk_cluster_queries.c, default to using sync dispatch instead of
starting a mainloop with timeout 30s that behaves basically like sync
dispatch.

This makes it easier to reason about calling these functions when the
caller may have its own mainloop.

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 include/pacemaker.h                  |  8 +++-
 lib/pacemaker/pcmk_cluster_queries.c | 64 ++++++++++++++++++++++------
 2 files changed, 57 insertions(+), 15 deletions(-)

diff --git a/include/pacemaker.h b/include/pacemaker.h
index a76569a..0ca9c29 100644
--- a/include/pacemaker.h
+++ b/include/pacemaker.h
@@ -111,7 +111,13 @@ void pcmk_free_injections(pcmk_injections_t *injections);
  *
  * \param[in,out] xml                 Destination for the result, as an XML tree
  * \param[in]     ipc_name            IPC name for request
- * \param[in]     message_timeout_ms  Message timeout
+ * \param[in]     message_timeout_ms  How long to wait for a reply from the
+ *                                    \p pacemakerd API. If 0,
+ *                                    \p pcmk_ipc_dispatch_sync will be used.
+ *                                    If positive, \p pcmk_ipc_dispatch_main
+ *                                    will be used, and a new mainloop will be
+ *                                    created for this purpose (freed before
+ *                                    return).
  *
  * \return Standard Pacemaker return code
  */
diff --git a/lib/pacemaker/pcmk_cluster_queries.c b/lib/pacemaker/pcmk_cluster_queries.c
index 00a809d..d4361c9 100644
--- a/lib/pacemaker/pcmk_cluster_queries.c
+++ b/lib/pacemaker/pcmk_cluster_queries.c
@@ -246,13 +246,13 @@ pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
 }
 
 static pcmk_ipc_api_t *
-ipc_connect(data_t *data, enum pcmk_ipc_server server, pcmk_ipc_callback_t cb)
+ipc_connect(data_t *data, enum pcmk_ipc_server server, pcmk_ipc_callback_t cb,
+            enum pcmk_ipc_dispatch dispatch_type)
 {
     int rc;
     pcmk__output_t *out = data->out;
     pcmk_ipc_api_t *api = NULL;
 
-
     rc = pcmk_new_ipc_api(&api, server);
     if (api == NULL) {
         out->err(out, "error: Could not connect to %s: %s",
@@ -264,7 +264,8 @@ ipc_connect(data_t *data, enum pcmk_ipc_server server, pcmk_ipc_callback_t cb)
     if (cb != NULL) {
         pcmk_register_ipc_callback(api, cb, data);
     }
-    rc = pcmk_connect_ipc(api, pcmk_ipc_dispatch_main);
+
+    rc = pcmk_connect_ipc(api, dispatch_type);
     if (rc != pcmk_rc_ok) {
         out->err(out, "error: Could not connect to %s: %s",
                 pcmk_ipc_name(api, true),
@@ -288,16 +289,26 @@ pcmk__controller_status(pcmk__output_t *out, char *dest_node, guint message_time
         .message_timeout_ms = message_timeout_ms,
         .pcmkd_state = pcmk_pacemakerd_state_invalid,
     };
-    pcmk_ipc_api_t *controld_api = ipc_connect(&data, pcmk_ipc_controld, controller_status_event_cb);
+    enum pcmk_ipc_dispatch dispatch_type = pcmk_ipc_dispatch_main;
+    pcmk_ipc_api_t *controld_api = NULL;
+
+    if (message_timeout_ms == 0) {
+        dispatch_type = pcmk_ipc_dispatch_sync;
+    }
+    controld_api = ipc_connect(&data, pcmk_ipc_controld,
+                               controller_status_event_cb, dispatch_type);
 
     if (controld_api != NULL) {
         int rc = pcmk_controld_api_ping(controld_api, dest_node);
         if (rc != pcmk_rc_ok) {
-            out->err(out, "error: Command failed: %s", pcmk_rc_str(rc));
+            out->err(out, "error: Could not ping controller API: %s",
+                     pcmk_rc_str(rc));
             data.rc = rc;
         }
 
-        start_main_loop(&data);
+        if (dispatch_type == pcmk_ipc_dispatch_main) {
+            start_main_loop(&data);
+        }
 
         pcmk_free_ipc_api(controld_api);
     }
@@ -334,16 +345,26 @@ pcmk__designated_controller(pcmk__output_t *out, guint message_timeout_ms)
         .message_timeout_ms = message_timeout_ms,
         .pcmkd_state = pcmk_pacemakerd_state_invalid,
     };
-    pcmk_ipc_api_t *controld_api = ipc_connect(&data, pcmk_ipc_controld, designated_controller_event_cb);
+    enum pcmk_ipc_dispatch dispatch_type = pcmk_ipc_dispatch_main;
+    pcmk_ipc_api_t *controld_api = NULL;
+
+    if (message_timeout_ms == 0) {
+        dispatch_type = pcmk_ipc_dispatch_sync;
+    }
+    controld_api = ipc_connect(&data, pcmk_ipc_controld,
+                               designated_controller_event_cb, dispatch_type);
 
     if (controld_api != NULL) {
         int rc = pcmk_controld_api_ping(controld_api, NULL);
         if (rc != pcmk_rc_ok) {
-            out->err(out, "error: Command failed: %s", pcmk_rc_str(rc));
+            out->err(out, "error: Could not ping controller API: %s",
+                     pcmk_rc_str(rc));
             data.rc = rc;
         }
 
-        start_main_loop(&data);
+        if (dispatch_type == pcmk_ipc_dispatch_main) {
+            start_main_loop(&data);
+        }
 
         pcmk_free_ipc_api(controld_api);
     }
@@ -375,7 +396,13 @@ pcmk_designated_controller(xmlNodePtr *xml, unsigned int message_timeout_ms)
  *
  * \param[in,out] out                 Output object
  * \param[in]     ipc_name            IPC name for request
- * \param[in]     message_timeout_ms  Message timeout
+ * \param[in]     message_timeout_ms  How long to wait for a reply from the
+ *                                    \p pacemakerd API. If 0,
+ *                                    \p pcmk_ipc_dispatch_sync will be used.
+ *                                    If positive, \p pcmk_ipc_dispatch_main
+ *                                    will be used, and a new mainloop will be
+ *                                    created for this purpose (freed before
+ *                                    return).
  * \param[out]    state               Where to store the \p pacemakerd state, if
  *                                    not \p NULL
  *
@@ -394,17 +421,26 @@ pcmk__pacemakerd_status(pcmk__output_t *out, const char *ipc_name,
         .message_timeout_ms = message_timeout_ms,
         .pcmkd_state = pcmk_pacemakerd_state_invalid,
     };
-    pcmk_ipc_api_t *pacemakerd_api = ipc_connect(&data, pcmk_ipc_pacemakerd, pacemakerd_event_cb);
+    enum pcmk_ipc_dispatch dispatch_type = pcmk_ipc_dispatch_main;
+    pcmk_ipc_api_t *pacemakerd_api = NULL;
+
+    if (message_timeout_ms == 0) {
+        dispatch_type = pcmk_ipc_dispatch_sync;
+    }
+    pacemakerd_api = ipc_connect(&data, pcmk_ipc_pacemakerd,
+                                 pacemakerd_event_cb, dispatch_type);
 
     if (pacemakerd_api != NULL) {
         int rc = pcmk_pacemakerd_api_ping(pacemakerd_api, ipc_name);
         if (rc != pcmk_rc_ok) {
-            out->err(out, "error: Command failed: %s", pcmk_rc_str(rc));
+            out->err(out, "error: Could not ping launcher API: %s",
+                     pcmk_rc_str(rc));
             data.rc = rc;
         }
 
-        start_main_loop(&data);
-
+        if (dispatch_type == pcmk_ipc_dispatch_main) {
+            start_main_loop(&data);
+        }
         pcmk_free_ipc_api(pacemakerd_api);
     }
 
-- 
2.31.1

From 97cb9452bb918c0b8ad6d1b937bff8f222191580 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Mon, 10 Oct 2022 18:10:54 -0700
Subject: [PATCH 21/22] Fix: tools: crm_mon --one-shot fails while pacemaker is
 shutting down

crm_mon --one-shot checks the pacemakerd state before trying to get a
CIB connection. If pacemakerd is shutting down, it returns ENOTCONN.
This can cause a resource agent that calls crm_mon (for example,
ocf:heartbeat:pgsql) to fail to stop during shutdown.

This is a regression introduced by commit 3f342e3.
crm_mon.c:pacemakerd_status() returns pcmk_rc_ok if pacemakerd is
shutting down, since 49ebe4c and 46d6edd (fixes for CLBZ#5471). 3f342e3
refactored crm_mon --one-shot to use library functions. pcmk__status()
now does most of the work, calling pcmk_status.c:pacemakerd_status().
That function returns ENOTCONN if pacemakerd is shutting down. As a
result, we don't try to connect to the CIB during shutdown.

Here we update pcmk__status() to use pcmk__pacemakerd_status() instead
of a static and mostly redundant pacemakerd_status(). It receives the
pacemakerd state via an output pointer argument. If pacemakerd is
running or shutting down (or if we get an EREMOTEIO rc), we try
connecting to the fencer and CIB. However, as long as we successfully
get the pacemakerd state, we return success from pcmk__status(), since
we did obtain the cluster status.

A couple of minor notes:
* pcmk__status() now takes a timeout argument that it passes to
  pcmk__pacemakerd_status(). timeout == 0 uses pcmk_ipc_dispatch_sync,
  matching the old implementation. A positive timeout uses
  pcmk_ipc_dispatch_main.
* pcmk_cluster_queries.c:ipc_connect() no longer always prints a "Could
  not connect" error for EREMOTEIO. The caller may consider it OK.

Fixes T579
Fixes CLBZ#5501

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 include/pcmki/pcmki_status.h         |   2 +-
 lib/pacemaker/pcmk_cluster_queries.c |  27 ++++--
 lib/pacemaker/pcmk_status.c          | 120 +++++++++------------------
 tools/crm_mon.c                      |   2 +-
 4 files changed, 61 insertions(+), 90 deletions(-)

diff --git a/include/pcmki/pcmki_status.h b/include/pcmki/pcmki_status.h
index 2bbd099..0dde21c 100644
--- a/include/pcmki/pcmki_status.h
+++ b/include/pcmki/pcmki_status.h
@@ -50,7 +50,7 @@ int pcmk__status(pcmk__output_t *out, cib_t *cib,
                  enum pcmk__fence_history fence_history, uint32_t show,
                  uint32_t show_opts, const char *only_node,
                  const char *only_rsc, const char *neg_location_prefix,
-                 bool simple_output);
+                 bool simple_output, guint timeout_ms);
 
 #ifdef __cplusplus
 }
diff --git a/lib/pacemaker/pcmk_cluster_queries.c b/lib/pacemaker/pcmk_cluster_queries.c
index d4361c9..220c872 100644
--- a/lib/pacemaker/pcmk_cluster_queries.c
+++ b/lib/pacemaker/pcmk_cluster_queries.c
@@ -247,7 +247,7 @@ pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
 
 static pcmk_ipc_api_t *
 ipc_connect(data_t *data, enum pcmk_ipc_server server, pcmk_ipc_callback_t cb,
-            enum pcmk_ipc_dispatch dispatch_type)
+            enum pcmk_ipc_dispatch dispatch_type, bool eremoteio_ok)
 {
     int rc;
     pcmk__output_t *out = data->out;
@@ -267,9 +267,15 @@ ipc_connect(data_t *data, enum pcmk_ipc_server server, pcmk_ipc_callback_t cb,
 
     rc = pcmk_connect_ipc(api, dispatch_type);
     if (rc != pcmk_rc_ok) {
-        out->err(out, "error: Could not connect to %s: %s",
-                pcmk_ipc_name(api, true),
-                pcmk_rc_str(rc));
+        if ((rc == EREMOTEIO) && eremoteio_ok) {
+            /* EREMOTEIO may be expected and acceptable for some callers.
+             * Preserve the return code in case callers need to handle it
+             * specially.
+             */
+        } else {
+            out->err(out, "error: Could not connect to %s: %s",
+                     pcmk_ipc_name(api, true), pcmk_rc_str(rc));
+        }
         data->rc = rc;
         pcmk_free_ipc_api(api);
         return NULL;
@@ -296,7 +302,8 @@ pcmk__controller_status(pcmk__output_t *out, char *dest_node, guint message_time
         dispatch_type = pcmk_ipc_dispatch_sync;
     }
     controld_api = ipc_connect(&data, pcmk_ipc_controld,
-                               controller_status_event_cb, dispatch_type);
+                               controller_status_event_cb, dispatch_type,
+                               false);
 
     if (controld_api != NULL) {
         int rc = pcmk_controld_api_ping(controld_api, dest_node);
@@ -352,7 +359,8 @@ pcmk__designated_controller(pcmk__output_t *out, guint message_timeout_ms)
         dispatch_type = pcmk_ipc_dispatch_sync;
     }
     controld_api = ipc_connect(&data, pcmk_ipc_controld,
-                               designated_controller_event_cb, dispatch_type);
+                               designated_controller_event_cb, dispatch_type,
+                               false);
 
     if (controld_api != NULL) {
         int rc = pcmk_controld_api_ping(controld_api, NULL);
@@ -407,6 +415,11 @@ pcmk_designated_controller(xmlNodePtr *xml, unsigned int message_timeout_ms)
  *                                    not \p NULL
  *
  * \return Standard Pacemaker return code
+ *
+ * \note This function returns \p EREMOTEIO if run on a Pacemaker Remote node
+ *       with \p pacemaker-remoted running, since \p pacemakerd is not proxied
+ *       to remote nodes. The fencer and CIB may still be accessible, but
+ *       \p state will be \p pcmk_pacemakerd_state_invalid.
  */
 int
 pcmk__pacemakerd_status(pcmk__output_t *out, const char *ipc_name,
@@ -428,7 +441,7 @@ pcmk__pacemakerd_status(pcmk__output_t *out, const char *ipc_name,
         dispatch_type = pcmk_ipc_dispatch_sync;
     }
     pacemakerd_api = ipc_connect(&data, pcmk_ipc_pacemakerd,
-                                 pacemakerd_event_cb, dispatch_type);
+                                 pacemakerd_event_cb, dispatch_type, true);
 
     if (pacemakerd_api != NULL) {
         int rc = pcmk_pacemakerd_api_ping(pacemakerd_api, ipc_name);
diff --git a/lib/pacemaker/pcmk_status.c b/lib/pacemaker/pcmk_status.c
index 1bf0172..794c9ea 100644
--- a/lib/pacemaker/pcmk_status.c
+++ b/lib/pacemaker/pcmk_status.c
@@ -70,71 +70,6 @@ fencing_connect(void)
     }
 }
 
-static void
-pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
-                    enum pcmk_ipc_event event_type, crm_exit_t status,
-                    void *event_data, void *user_data)
-{
-    pcmk_pacemakerd_api_reply_t *reply = event_data;
-    enum pcmk_pacemakerd_state *state =
-        (enum pcmk_pacemakerd_state *) user_data;
-
-    /* we are just interested in the latest reply */
-    *state = pcmk_pacemakerd_state_invalid;
-
-    if (event_type != pcmk_ipc_event_reply || status != CRM_EX_OK) {
-        return;
-    }
-
-    if (reply->reply_type == pcmk_pacemakerd_reply_ping &&
-        reply->data.ping.last_good != (time_t) 0 &&
-        reply->data.ping.status == pcmk_rc_ok) {
-        *state = reply->data.ping.state;
-    }
-}
-
-static int
-pacemakerd_status(pcmk__output_t *out)
-{
-    int rc = pcmk_rc_ok;
-    pcmk_ipc_api_t *pacemakerd_api = NULL;
-    enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid;
-
-    rc = pcmk_new_ipc_api(&pacemakerd_api, pcmk_ipc_pacemakerd);
-    if (pacemakerd_api == NULL) {
-        out->err(out, "Could not connect to pacemakerd: %s",
-                 pcmk_rc_str(rc));
-        return rc;
-    }
-
-    pcmk_register_ipc_callback(pacemakerd_api, pacemakerd_event_cb, (void *) &state);
-
-    rc = pcmk_connect_ipc(pacemakerd_api, pcmk_ipc_dispatch_sync);
-    if (rc == EREMOTEIO) {
-        return pcmk_rc_ok;
-    } else if (rc != pcmk_rc_ok) {
-        out->err(out, "Could not connect to pacemakerd: %s",
-                 pcmk_rc_str(rc));
-        pcmk_free_ipc_api(pacemakerd_api);
-        return rc;
-    }
-
-    rc = pcmk_pacemakerd_api_ping(pacemakerd_api, crm_system_name);
-
-    if (rc != pcmk_rc_ok) {
-        /* Got some error from pcmk_pacemakerd_api_ping, so return it. */
-    } else if (state == pcmk_pacemakerd_state_running) {
-        rc = pcmk_rc_ok;
-    } else if (state == pcmk_pacemakerd_state_shutting_down) {
-        rc = ENOTCONN;
-    } else {
-        rc = EAGAIN;
-    }
-
-    pcmk_free_ipc_api(pacemakerd_api);
-    return rc;
-}
-
 /*!
  * \internal
  * \brief Output the cluster status given a fencer and CIB connection
@@ -256,7 +191,7 @@ pcmk_status(xmlNodePtr *xml)
     stonith__register_messages(out);
 
     rc = pcmk__status(out, cib, pcmk__fence_history_full, pcmk_section_all,
-                      show_opts, NULL, NULL, NULL, false);
+                      show_opts, NULL, NULL, NULL, false, 0);
     pcmk__out_epilogue(out, xml, rc);
 
     cib_delete(cib);
@@ -288,6 +223,13 @@ pcmk_status(xmlNodePtr *xml)
  * \param[in]     simple_output        Whether to use a simple output format.
  *                                     Note: This is for use by \p crm_mon only
  *                                     and is planned to be deprecated.
+ * \param[in]     timeout_ms           How long to wait for a reply from the
+ *                                     \p pacemakerd API. If 0,
+ *                                     \p pcmk_ipc_dispatch_sync will be used.
+ *                                     If positive, \p pcmk_ipc_dispatch_main
+ *                                     will be used, and a new mainloop will be
+ *                                     created for this purpose (freed before
+ *                                     return).
  *
  * \return Standard Pacemaker return code
  */
@@ -295,34 +237,47 @@ int
 pcmk__status(pcmk__output_t *out, cib_t *cib,
              enum pcmk__fence_history fence_history, uint32_t show,
              uint32_t show_opts, const char *only_node, const char *only_rsc,
-             const char *neg_location_prefix, bool simple_output)
+             const char *neg_location_prefix, bool simple_output,
+             guint timeout_ms)
 {
     xmlNode *current_cib = NULL;
     int rc = pcmk_rc_ok;
     stonith_t *stonith = NULL;
+    enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid;
 
     if (cib == NULL) {
         return ENOTCONN;
     }
 
-    if (cib->variant == cib_native) {
-        if (cib->state == cib_connected_query || cib->state == cib_connected_command) {
-            rc = pcmk_rc_ok;
-        } else {
-            rc = pacemakerd_status(out);
+    if ((cib->variant == cib_native)
+        && (cib->state != cib_connected_query)
+        && (cib->state != cib_connected_command)) {
+
+        rc = pcmk__pacemakerd_status(out, crm_system_name, timeout_ms, &state);
+        switch (rc) {
+            case pcmk_rc_ok:
+                switch (state) {
+                    case pcmk_pacemakerd_state_running:
+                    case pcmk_pacemakerd_state_shutting_down:
+                        // CIB may still be available while shutting down
+                        break;
+                    default:
+                        return rc;
+                }
+                break;
+            case EREMOTEIO:
+                /* We'll always get EREMOTEIO if we run this on a Pacemaker
+                 * Remote node. The fencer and CIB might be available.
+                 */
+                rc = pcmk_rc_ok;
+                break;
+            default:
+                return rc;
         }
     }
 
-    if (rc != pcmk_rc_ok) {
-        return rc;
-    }
-
     if (fence_history != pcmk__fence_history_none && cib->variant == cib_native) {
         stonith = fencing_connect();
-
-        if (stonith == NULL) {
-            return ENOTCONN;
-        }
     }
 
     rc = cib_connect(out, cib, &current_cib);
@@ -334,6 +289,9 @@ pcmk__status(pcmk__output_t *out, cib_t *cib,
                                      fence_history, show, show_opts, only_node,
                                      only_rsc, neg_location_prefix,
                                      simple_output);
+    if (rc != pcmk_rc_ok) {
+        out->err(out, "Error outputting status info from the fencer or CIB");
+    }
 
 done:
     if (stonith != NULL) {
@@ -345,7 +303,7 @@ done:
         stonith_api_delete(stonith);
     }
 
-    return rc;
+    return pcmk_rc_ok;
 }
 
 /* This is an internal-only function that is planned to be deprecated and removed.
diff --git a/tools/crm_mon.c b/tools/crm_mon.c
index e8cb709..c70c439 100644
--- a/tools/crm_mon.c
+++ b/tools/crm_mon.c
@@ -1330,7 +1330,7 @@ one_shot(void)
     int rc = pcmk__status(out, cib, fence_history, show, show_opts,
                           options.only_node, options.only_rsc,
                           options.neg_location_prefix,
-                          output_format == mon_output_monitor);
+                          output_format == mon_output_monitor, 0);
 
     if (rc == pcmk_rc_ok) {
         clean_up(pcmk_rc2exitc(rc));
-- 
2.31.1

From 4e63214f61f03d2756f884dd411db07cb22e9de6 Mon Sep 17 00:00:00 2001
From: Reid Wahl <nrwahl@protonmail.com>
Date: Tue, 11 Oct 2022 13:25:45 -0700
Subject: [PATCH 22/22] Low: libpacemaker: Correct sys_from default in
 pacemakerd_health()

sys_from should be a subsystem ("pacemakerd" is expected), not a node.

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
---
 lib/pacemaker/pcmk_output.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pacemaker/pcmk_output.c b/lib/pacemaker/pcmk_output.c
index 153a422..b61f354 100644
--- a/lib/pacemaker/pcmk_output.c
+++ b/lib/pacemaker/pcmk_output.c
@@ -642,7 +642,7 @@ pacemakerd_health(pcmk__output_t *out, va_list args)
         state_s = pcmk__pcmkd_state_enum2friendly(state);
     }
     return out->info(out, "Status of %s: '%s' (last updated %s)",
-                     (!pcmk__str_empty(sys_from)) ? sys_from : "unknown node",
+                     (!pcmk__str_empty(sys_from)) ? sys_from : "unknown subsystem",
                      state_s,
                      (!pcmk__str_empty(last_updated)) ? last_updated : "at unknown time");
 }
@@ -664,7 +664,7 @@ pacemakerd_health_html(pcmk__output_t *out, va_list args)
     }
 
     msg = crm_strdup_printf("Status of %s: '%s' (last updated %s)",
-                            (!pcmk__str_empty(sys_from)) ? sys_from : "unknown node",
+                            (!pcmk__str_empty(sys_from)) ? sys_from : "unknown subsystem",
                             state_s,
                             (!pcmk__str_empty(last_updated)) ? last_updated : "at unknown time");
     pcmk__output_create_html_node(out, "li", NULL, NULL, msg);
-- 
2.31.1