945d2e
From e34421b2608f235c6a77eb6de4596d93a2128be1 Mon Sep 17 00:00:00 2001
945d2e
From: Ken Gaillot <kgaillot@redhat.com>
945d2e
Date: Fri, 3 Apr 2020 11:13:52 -0500
945d2e
Subject: [PATCH 1/6] Refactor: libcrmcommon: new model for daemon IPC API
945d2e
945d2e
This is based on tools/crm_resource_controller.[ch] and the node removal code
945d2e
in tools/crm_node.c, but generalized for any daemon. As of this commit, no
945d2e
specific daemon is supported.
945d2e
---
945d2e
 include/crm/common/internal.h  |   8 +
945d2e
 include/crm/common/ipc.h       |  95 +++++-
945d2e
 lib/common/crmcommon_private.h |  86 +++++
945d2e
 lib/common/ipc_client.c        | 703 ++++++++++++++++++++++++++++++++++++++++-
945d2e
 lib/common/mainloop.c          |  70 ++--
945d2e
 5 files changed, 940 insertions(+), 22 deletions(-)
945d2e
945d2e
diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h
945d2e
index 28b20b4..13c29ea 100644
945d2e
--- a/include/crm/common/internal.h
945d2e
+++ b/include/crm/common/internal.h
945d2e
@@ -19,6 +19,7 @@
945d2e
 #include <libxml/tree.h>        // xmlNode
945d2e
 
945d2e
 #include <crm/common/util.h>    // crm_strdup_printf()
945d2e
+#include <crm/common/mainloop.h> // mainloop_io_t, struct ipc_client_callbacks
945d2e
 
945d2e
 // Internal ACL-related utilities (from acl.c)
945d2e
 
945d2e
@@ -103,6 +104,13 @@ pcmk__open_devnull(int flags)
945d2e
     } while (0)
945d2e
 
945d2e
 
945d2e
+/* internal main loop utilities (from mainloop.c) */
945d2e
+
945d2e
+int pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata,
945d2e
+                           struct ipc_client_callbacks *callbacks,
945d2e
+                           mainloop_io_t **source);
945d2e
+
945d2e
+
945d2e
 /* internal procfs utilities (from procfs.c) */
945d2e
 
945d2e
 pid_t pcmk__procfs_pid_of(const char *name);
945d2e
diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h
945d2e
index a0df956..8dee1b1 100644
945d2e
--- a/include/crm/common/ipc.h
945d2e
+++ b/include/crm/common/ipc.h
945d2e
@@ -16,7 +16,8 @@ extern "C" {
945d2e
 
945d2e
 /**
945d2e
  * \file
945d2e
- * \brief Wrappers for and extensions to libqb IPC
945d2e
+ * \brief IPC interface to Pacemaker daemons
945d2e
+ *
945d2e
  * \ingroup core
945d2e
  */
945d2e
 
945d2e
@@ -48,6 +49,96 @@ xmlNode *create_request_adv(const char *task, xmlNode *xml_data,
945d2e
                             const char *origin);
945d2e
 
945d2e
 
945d2e
+/*
945d2e
+ * The library supports two methods of creating IPC connections. The older code
945d2e
+ * allows connecting to any arbitrary IPC name. The newer code only allows
945d2e
+ * connecting to one of the Pacemaker daemons.
945d2e
+ *
945d2e
+ * As daemons are converted to use the new model, the old functions should be
945d2e
+ * considered deprecated for use with those daemons. Once all daemons are
945d2e
+ * converted, the old functions should be officially deprecated as public API
945d2e
+ * and eventually made internal API.
945d2e
+ */
945d2e
+
945d2e
+/*
945d2e
+ * Pacemaker daemon IPC
945d2e
+ */
945d2e
+
945d2e
+//! Available IPC interfaces
945d2e
+enum pcmk_ipc_server {
945d2e
+    pcmk_ipc_attrd,         //!< Attribute manager
945d2e
+    pcmk_ipc_based,         //!< CIB manager
945d2e
+    pcmk_ipc_controld,      //!< Controller
945d2e
+    pcmk_ipc_execd,         //!< Executor
945d2e
+    pcmk_ipc_fenced,        //!< Fencer
945d2e
+    pcmk_ipc_pacemakerd,    //!< Launcher
945d2e
+    pcmk_ipc_schedulerd,    //!< Scheduler
945d2e
+};
945d2e
+
945d2e
+//! Possible event types that an IPC event callback can be called for
945d2e
+enum pcmk_ipc_event {
945d2e
+    pcmk_ipc_event_connect,     //!< Result of asynchronous connection attempt
945d2e
+    pcmk_ipc_event_disconnect,  //!< Termination of IPC connection
945d2e
+    pcmk_ipc_event_reply,       //!< Daemon's reply to client IPC request
945d2e
+    pcmk_ipc_event_notify,      //!< Notification from daemon
945d2e
+};
945d2e
+
945d2e
+//! How IPC replies should be dispatched
945d2e
+enum pcmk_ipc_dispatch {
945d2e
+    pcmk_ipc_dispatch_main, //!< Attach IPC to GMainLoop for dispatch
945d2e
+    pcmk_ipc_dispatch_poll, //!< Caller will poll and dispatch IPC
945d2e
+    pcmk_ipc_dispatch_sync, //!< Sending a command will wait for any reply
945d2e
+};
945d2e
+
945d2e
+//! Client connection to Pacemaker IPC
945d2e
+typedef struct pcmk_ipc_api_s pcmk_ipc_api_t;
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Callback function type for Pacemaker daemon IPC APIs
945d2e
+ *
945d2e
+ * \param[in] api         IPC API connection
945d2e
+ * \param[in] event_type  The type of event that occurred
945d2e
+ * \param[in] status      Event status
945d2e
+ * \param[in] event_data  Event-specific data
945d2e
+ * \param[in] user_data   Caller data provided when callback was registered
945d2e
+ *
945d2e
+ * \note For connection and disconnection events, event_data may be NULL (for
945d2e
+ *       local IPC) or the name of the connected node (for remote IPC, for
945d2e
+ *       daemons that support that). For reply and notify events, event_data is
945d2e
+ *       defined by the specific daemon API.
945d2e
+ */
945d2e
+typedef void (*pcmk_ipc_callback_t)(pcmk_ipc_api_t *api,
945d2e
+                                    enum pcmk_ipc_event event_type,
945d2e
+                                    crm_exit_t status,
945d2e
+                                    void *event_data, void *user_data);
945d2e
+
945d2e
+int pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server);
945d2e
+
945d2e
+void pcmk_free_ipc_api(pcmk_ipc_api_t *api);
945d2e
+
945d2e
+int pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type);
945d2e
+
945d2e
+void pcmk_disconnect_ipc(pcmk_ipc_api_t *api);
945d2e
+
945d2e
+int pcmk_poll_ipc(pcmk_ipc_api_t *api, int timeout_ms);
945d2e
+
945d2e
+void pcmk_dispatch_ipc(pcmk_ipc_api_t *api);
945d2e
+
945d2e
+void pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb,
945d2e
+                                void *user_data);
945d2e
+
945d2e
+const char *pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log);
945d2e
+
945d2e
+bool pcmk_ipc_is_connected(pcmk_ipc_api_t *api);
945d2e
+
945d2e
+int pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name,
945d2e
+                        uint32_t nodeid);
945d2e
+
945d2e
+
945d2e
+/*
945d2e
+ * Generic IPC API (to eventually be deprecated as public API and made internal)
945d2e
+ */
945d2e
+
945d2e
 /* *INDENT-OFF* */
945d2e
 enum crm_ipc_flags
945d2e
 {
945d2e
@@ -58,7 +149,7 @@ enum crm_ipc_flags
945d2e
     crm_ipc_proxied         = 0x00000100, /* _ALL_ replies to proxied connections need to be sent as events */
945d2e
     crm_ipc_client_response = 0x00000200, /* A Response is expected in reply */
945d2e
 
945d2e
-    // These are options only for pcmk__ipc_send_iov()
945d2e
+    // These are options for Pacemaker's internal use only (pcmk__ipc_send_*())
945d2e
     crm_ipc_server_event    = 0x00010000, /* Send an Event instead of a Response */
945d2e
     crm_ipc_server_free     = 0x00020000, /* Free the iovec after sending */
945d2e
     crm_ipc_proxied_relay_response = 0x00040000, /* all replies to proxied connections are sent as events, this flag preserves whether the event should be treated as an actual event, or a response.*/
945d2e
diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h
945d2e
index d06fa20..f9df27d 100644
945d2e
--- a/lib/common/crmcommon_private.h
945d2e
+++ b/lib/common/crmcommon_private.h
945d2e
@@ -103,6 +103,84 @@ pcmk__xml_attr_value(const xmlAttr *attr)
945d2e
 
945d2e
 #define PCMK__IPC_VERSION 1
945d2e
 
945d2e
+// IPC behavior that varies by daemon
945d2e
+typedef struct pcmk__ipc_methods_s {
945d2e
+    /*!
945d2e
+     * \internal
945d2e
+     * \brief Allocate any private data needed by daemon IPC
945d2e
+     *
945d2e
+     * \param[in] api  IPC API connection
945d2e
+     *
945d2e
+     * \return Standard Pacemaker return code
945d2e
+     */
945d2e
+    int (*new_data)(pcmk_ipc_api_t *api);
945d2e
+
945d2e
+    /*!
945d2e
+     * \internal
945d2e
+     * \brief Free any private data used by daemon IPC
945d2e
+     *
945d2e
+     * \param[in] api_data  Data allocated by new_data() method
945d2e
+     */
945d2e
+    void (*free_data)(void *api_data);
945d2e
+
945d2e
+    /*!
945d2e
+     * \internal
945d2e
+     * \brief Perform daemon-specific handling after successful connection
945d2e
+     *
945d2e
+     * Some daemons require clients to register before sending any other
945d2e
+     * commands. The controller requires a CRM_OP_HELLO (with no reply), and
945d2e
+     * the CIB manager, executor, and fencer require a CRM_OP_REGISTER (with a
945d2e
+     * reply). Ideally this would be consistent across all daemons, but for now
945d2e
+     * this allows each to do its own authorization.
945d2e
+     *
945d2e
+     * \param[in] api  IPC API connection
945d2e
+     *
945d2e
+     * \return Standard Pacemaker return code
945d2e
+     */
945d2e
+    int (*post_connect)(pcmk_ipc_api_t *api);
945d2e
+
945d2e
+    /*!
945d2e
+     * \internal
945d2e
+     * \brief Check whether an IPC request results in a reply
945d2e
+     *
945d2e
+     * \parma[in] api      IPC API connection
945d2e
+     * \param[in] request  IPC request XML
945d2e
+     *
945d2e
+     * \return true if request would result in an IPC reply, false otherwise
945d2e
+     */
945d2e
+    bool (*reply_expected)(pcmk_ipc_api_t *api, xmlNode *request);
945d2e
+
945d2e
+    /*!
945d2e
+     * \internal
945d2e
+     * \brief Perform daemon-specific handling of an IPC message
945d2e
+     *
945d2e
+     * \param[in] api  IPC API connection
945d2e
+     * \param[in] msg  Message read from IPC connection
945d2e
+     */
945d2e
+    void (*dispatch)(pcmk_ipc_api_t *api, xmlNode *msg);
945d2e
+
945d2e
+    /*!
945d2e
+     * \internal
945d2e
+     * \brief Perform daemon-specific handling of an IPC disconnect
945d2e
+     *
945d2e
+     * \param[in] api  IPC API connection
945d2e
+     */
945d2e
+    void (*post_disconnect)(pcmk_ipc_api_t *api);
945d2e
+} pcmk__ipc_methods_t;
945d2e
+
945d2e
+// Implementation of pcmk_ipc_api_t
945d2e
+struct pcmk_ipc_api_s {
945d2e
+    enum pcmk_ipc_server server;          // Daemon this IPC API instance is for
945d2e
+    enum pcmk_ipc_dispatch dispatch_type; // How replies should be dispatched
945d2e
+    crm_ipc_t *ipc;                       // IPC connection
945d2e
+    mainloop_io_t *mainloop_io;     // If using mainloop, I/O source for IPC
945d2e
+    bool free_on_disconnect;        // Whether disconnect should free object
945d2e
+    pcmk_ipc_callback_t cb;         // Caller-registered callback (if any)
945d2e
+    void *user_data;                // Caller-registered data (if any)
945d2e
+    void *api_data;                 // For daemon-specific use
945d2e
+    pcmk__ipc_methods_t *cmds;      // Behavior that varies by daemon
945d2e
+};
945d2e
+
945d2e
 typedef struct pcmk__ipc_header_s {
945d2e
     struct qb_ipc_response_header qb;
945d2e
     uint32_t size_uncompressed;
945d2e
@@ -112,6 +190,14 @@ typedef struct pcmk__ipc_header_s {
945d2e
 } pcmk__ipc_header_t;
945d2e
 
945d2e
 G_GNUC_INTERNAL
945d2e
+int pcmk__send_ipc_request(pcmk_ipc_api_t *api, xmlNode *request);
945d2e
+
945d2e
+G_GNUC_INTERNAL
945d2e
+void pcmk__call_ipc_callback(pcmk_ipc_api_t *api,
945d2e
+                             enum pcmk_ipc_event event_type,
945d2e
+                             crm_exit_t status, void *event_data);
945d2e
+
945d2e
+G_GNUC_INTERNAL
945d2e
 unsigned int pcmk__ipc_buffer_size(unsigned int max);
945d2e
 
945d2e
 G_GNUC_INTERNAL
945d2e
diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c
945d2e
index 7737588..16dc9b5 100644
945d2e
--- a/lib/common/ipc_client.c
945d2e
+++ b/lib/common/ipc_client.c
945d2e
@@ -31,6 +31,679 @@
945d2e
 #include <crm/common/ipc_internal.h>
945d2e
 #include "crmcommon_private.h"
945d2e
 
945d2e
+/*!
945d2e
+ * \brief Create a new object for using Pacemaker daemon IPC
945d2e
+ *
945d2e
+ * \param[out] api     Where to store new IPC object
945d2e
+ * \param[in]  server  Which Pacemaker daemon the object is for
945d2e
+ *
945d2e
+ * \return Standard Pacemaker result code
945d2e
+ *
945d2e
+ * \note The caller is responsible for freeing *api using pcmk_free_ipc_api().
945d2e
+ * \note This is intended to supersede crm_ipc_new() but is not yet usable.
945d2e
+ */
945d2e
+int
945d2e
+pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
945d2e
+{
945d2e
+    size_t max_size = 0;
945d2e
+
945d2e
+    if (api == NULL) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+
945d2e
+    *api = calloc(1, sizeof(pcmk_ipc_api_t));
945d2e
+    if (*api == NULL) {
945d2e
+        return errno;
945d2e
+    }
945d2e
+
945d2e
+    (*api)->server = server;
945d2e
+    if (pcmk_ipc_name(*api, false) == NULL) {
945d2e
+        pcmk_free_ipc_api(*api);
945d2e
+        *api = NULL;
945d2e
+        return EOPNOTSUPP;
945d2e
+    }
945d2e
+
945d2e
+    // Set server methods and max_size (if not default)
945d2e
+    switch (server) {
945d2e
+        case pcmk_ipc_attrd:
945d2e
+            break;
945d2e
+
945d2e
+        case pcmk_ipc_based:
945d2e
+            max_size = 512 * 1024; // 512KB
945d2e
+            break;
945d2e
+
945d2e
+        case pcmk_ipc_controld:
945d2e
+            break;
945d2e
+
945d2e
+        case pcmk_ipc_execd:
945d2e
+            break;
945d2e
+
945d2e
+        case pcmk_ipc_fenced:
945d2e
+            break;
945d2e
+
945d2e
+        case pcmk_ipc_pacemakerd:
945d2e
+            break;
945d2e
+
945d2e
+        case pcmk_ipc_schedulerd:
945d2e
+            // @TODO max_size could vary by client, maybe take as argument?
945d2e
+            max_size = 5 * 1024 * 1024; // 5MB
945d2e
+            break;
945d2e
+    }
945d2e
+    if ((*api)->cmds == NULL) {
945d2e
+        pcmk_free_ipc_api(*api);
945d2e
+        *api = NULL;
945d2e
+        return ENOMEM;
945d2e
+    }
945d2e
+
945d2e
+    (*api)->ipc = crm_ipc_new(pcmk_ipc_name(*api, false), max_size);
945d2e
+    if ((*api)->ipc == NULL) {
945d2e
+        pcmk_free_ipc_api(*api);
945d2e
+        *api = NULL;
945d2e
+        return ENOMEM;
945d2e
+    }
945d2e
+
945d2e
+    // If daemon API has its own data to track, allocate it
945d2e
+    if ((*api)->cmds->new_data != NULL) {
945d2e
+        if ((*api)->cmds->new_data(*api) != pcmk_rc_ok) {
945d2e
+            pcmk_free_ipc_api(*api);
945d2e
+            *api = NULL;
945d2e
+            return ENOMEM;
945d2e
+        }
945d2e
+    }
945d2e
+    crm_trace("Created %s API IPC object", pcmk_ipc_name(*api, true));
945d2e
+    return pcmk_rc_ok;
945d2e
+}
945d2e
+
945d2e
+static void
945d2e
+free_daemon_specific_data(pcmk_ipc_api_t *api)
945d2e
+{
945d2e
+    if ((api != NULL) && (api->cmds != NULL)) {
945d2e
+        if ((api->cmds->free_data != NULL) && (api->api_data != NULL)) {
945d2e
+            api->cmds->free_data(api->api_data);
945d2e
+            api->api_data = NULL;
945d2e
+        }
945d2e
+        free(api->cmds);
945d2e
+        api->cmds = NULL;
945d2e
+    }
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \internal
945d2e
+ * \brief Call an IPC API event callback, if one is registed
945d2e
+ *
945d2e
+ * \param[in] api         IPC API connection
945d2e
+ * \param[in] event_type  The type of event that occurred
945d2e
+ * \param[in] status      Event status
945d2e
+ * \param[in] event_data  Event-specific data
945d2e
+ */
945d2e
+void
945d2e
+pcmk__call_ipc_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
945d2e
+                        crm_exit_t status, void *event_data)
945d2e
+{
945d2e
+    if ((api != NULL) && (api->cb != NULL)) {
945d2e
+        api->cb(api, event_type, status, event_data, api->user_data);
945d2e
+    }
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \internal
945d2e
+ * \brief Clean up after an IPC disconnect
945d2e
+ *
945d2e
+ * \param[in]  user_data  IPC API connection that disconnected
945d2e
+ *
945d2e
+ * \note This function can be used as a main loop IPC destroy callback.
945d2e
+ */
945d2e
+static void
945d2e
+ipc_post_disconnect(gpointer user_data)
945d2e
+{
945d2e
+    pcmk_ipc_api_t *api = user_data;
945d2e
+
945d2e
+    crm_info("Disconnected from %s IPC API", pcmk_ipc_name(api, true));
945d2e
+
945d2e
+    // Perform any daemon-specific handling needed
945d2e
+    if ((api->cmds != NULL) && (api->cmds->post_disconnect != NULL)) {
945d2e
+        api->cmds->post_disconnect(api);
945d2e
+    }
945d2e
+
945d2e
+    // Call client's registered event callback
945d2e
+    pcmk__call_ipc_callback(api, pcmk_ipc_event_disconnect, CRM_EX_DISCONNECT,
945d2e
+                            NULL);
945d2e
+
945d2e
+    /* If this is being called from a running main loop, mainloop_gio_destroy()
945d2e
+     * will free ipc and mainloop_io immediately after calling this function.
945d2e
+     * If this is called from a stopped main loop, these will leak, so the best
945d2e
+     * practice is to close the connection before stopping the main loop.
945d2e
+     */
945d2e
+    api->ipc = NULL;
945d2e
+    api->mainloop_io = NULL;
945d2e
+
945d2e
+    if (api->free_on_disconnect) {
945d2e
+        /* pcmk_free_ipc_api() has already been called, but did not free api
945d2e
+         * or api->cmds because this function needed them. Do that now.
945d2e
+         */
945d2e
+        free_daemon_specific_data(api);
945d2e
+        crm_trace("Freeing IPC API object after disconnect");
945d2e
+        free(api);
945d2e
+    }
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Free the contents of an IPC API object
945d2e
+ *
945d2e
+ * \param[in] api  IPC API object to free
945d2e
+ */
945d2e
+void
945d2e
+pcmk_free_ipc_api(pcmk_ipc_api_t *api)
945d2e
+{
945d2e
+    bool free_on_disconnect = false;
945d2e
+
945d2e
+    if (api == NULL) {
945d2e
+        return;
945d2e
+    }
945d2e
+    crm_debug("Releasing %s IPC API", pcmk_ipc_name(api, true));
945d2e
+
945d2e
+    if (api->ipc != NULL) {
945d2e
+        if (api->mainloop_io != NULL) {
945d2e
+            /* We need to keep the api pointer itself around, because it is the
945d2e
+             * user data for the IPC client destroy callback. That will be
945d2e
+             * triggered by the pcmk_disconnect_ipc() call below, but it might
945d2e
+             * happen later in the main loop (if still running).
945d2e
+             *
945d2e
+             * This flag tells the destroy callback to free the object. It can't
945d2e
+             * do that unconditionally, because the application might call this
945d2e
+             * function after a disconnect that happened by other means.
945d2e
+             */
945d2e
+            free_on_disconnect = api->free_on_disconnect = true;
945d2e
+        }
945d2e
+        pcmk_disconnect_ipc(api); // Frees api if free_on_disconnect is true
945d2e
+    }
945d2e
+    if (!free_on_disconnect) {
945d2e
+        free_daemon_specific_data(api);
945d2e
+        crm_trace("Freeing IPC API object");
945d2e
+        free(api);
945d2e
+    }
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Get the IPC name used with an IPC API connection
945d2e
+ *
945d2e
+ * \param[in] api      IPC API connection
945d2e
+ * \param[in] for_log  If true, return human-friendly name instead of IPC name
945d2e
+ *
945d2e
+ * \return IPC API's human-friendly or connection name, or if none is available,
945d2e
+ *         "Pacemaker" if for_log is true and NULL if for_log is false
945d2e
+ */
945d2e
+const char *
945d2e
+pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log)
945d2e
+{
945d2e
+    if (api == NULL) {
945d2e
+        return for_log? "Pacemaker" : NULL;
945d2e
+    }
945d2e
+    switch (api->server) {
945d2e
+        case pcmk_ipc_attrd:
945d2e
+            return for_log? "attribute manager" : NULL /* T_ATTRD */;
945d2e
+
945d2e
+        case pcmk_ipc_based:
945d2e
+            return for_log? "CIB manager" : NULL /* PCMK__SERVER_BASED_RW */;
945d2e
+
945d2e
+        case pcmk_ipc_controld:
945d2e
+            return for_log? "controller" : NULL /* CRM_SYSTEM_CRMD */;
945d2e
+
945d2e
+        case pcmk_ipc_execd:
945d2e
+            return for_log? "executor" : NULL /* CRM_SYSTEM_LRMD */;
945d2e
+
945d2e
+        case pcmk_ipc_fenced:
945d2e
+            return for_log? "fencer" : NULL /* "stonith-ng" */;
945d2e
+
945d2e
+        case pcmk_ipc_pacemakerd:
945d2e
+            return for_log? "launcher" : NULL /* CRM_SYSTEM_MCP */;
945d2e
+
945d2e
+        case pcmk_ipc_schedulerd:
945d2e
+            return for_log? "scheduler" : NULL /* CRM_SYSTEM_PENGINE */;
945d2e
+
945d2e
+        default:
945d2e
+            return for_log? "Pacemaker" : NULL;
945d2e
+    }
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Check whether an IPC API connection is active
945d2e
+ *
945d2e
+ * \param[in] api  IPC API connection
945d2e
+ *
945d2e
+ * \return true if IPC is connected, false otherwise
945d2e
+ */
945d2e
+bool
945d2e
+pcmk_ipc_is_connected(pcmk_ipc_api_t *api)
945d2e
+{
945d2e
+    return (api != NULL) && crm_ipc_connected(api->ipc);
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \internal
945d2e
+ * \brief Call the daemon-specific API's dispatch function
945d2e
+ *
945d2e
+ * Perform daemon-specific handling of IPC reply dispatch. It is the daemon
945d2e
+ * method's responsibility to call the client's registered event callback, as
945d2e
+ * well as allocate and free any event data.
945d2e
+ *
945d2e
+ * \param[in] api  IPC API connection
945d2e
+ */
945d2e
+static void
945d2e
+call_api_dispatch(pcmk_ipc_api_t *api, xmlNode *message)
945d2e
+{
945d2e
+    crm_log_xml_trace(message, "ipc-received");
945d2e
+    if ((api->cmds != NULL) && (api->cmds->dispatch != NULL)) {
945d2e
+        api->cmds->dispatch(api, message);
945d2e
+    }
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \internal
945d2e
+ * \brief Dispatch data read from IPC source
945d2e
+ *
945d2e
+ * \param[in] buffer     Data read from IPC
945d2e
+ * \param[in] length     Number of bytes of data in buffer (ignored)
945d2e
+ * \param[in] user_data  IPC object
945d2e
+ *
945d2e
+ * \return Always 0 (meaning connection is still required)
945d2e
+ *
945d2e
+ * \note This function can be used as a main loop IPC dispatch callback.
945d2e
+ */
945d2e
+static int
945d2e
+dispatch_ipc_data(const char *buffer, ssize_t length, gpointer user_data)
945d2e
+{
945d2e
+    pcmk_ipc_api_t *api = user_data;
945d2e
+    xmlNode *msg;
945d2e
+
945d2e
+    CRM_CHECK(api != NULL, return 0);
945d2e
+
945d2e
+    if (buffer == NULL) {
945d2e
+        crm_warn("Empty message received from %s IPC",
945d2e
+                 pcmk_ipc_name(api, true));
945d2e
+        return 0;
945d2e
+    }
945d2e
+
945d2e
+    msg = string2xml(buffer);
945d2e
+    if (msg == NULL) {
945d2e
+        crm_warn("Malformed message received from %s IPC",
945d2e
+                 pcmk_ipc_name(api, true));
945d2e
+        return 0;
945d2e
+    }
945d2e
+    call_api_dispatch(api, msg);
945d2e
+    free_xml(msg);
945d2e
+    return 0;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Check whether an IPC connection has data available (without main loop)
945d2e
+ *
945d2e
+ * \param[in]  api         IPC API connection
945d2e
+ * \param[in]  timeout_ms  If less than 0, poll indefinitely; if 0, poll once
945d2e
+ *                         and return immediately; otherwise, poll for up to
945d2e
+ *                         this many milliseconds
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ *
945d2e
+ * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
945d2e
+ *       this function to check whether IPC data is available. Return values of
945d2e
+ *       interest include pcmk_rc_ok meaning data is available, and EAGAIN
945d2e
+ *       meaning no data is available; all other values indicate errors.
945d2e
+ * \todo This does not allow the caller to poll multiple file descriptors at
945d2e
+ *       once. If there is demand for that, we could add a wrapper for
945d2e
+ *       crm_ipc_get_fd(api->ipc), so the caller can call poll() themselves.
945d2e
+ */
945d2e
+int
945d2e
+pcmk_poll_ipc(pcmk_ipc_api_t *api, int timeout_ms)
945d2e
+{
945d2e
+    int rc;
945d2e
+    struct pollfd pollfd = { 0, };
945d2e
+
945d2e
+    if ((api == NULL) || (api->dispatch_type != pcmk_ipc_dispatch_poll)) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+    pollfd.fd = crm_ipc_get_fd(api->ipc);
945d2e
+    pollfd.events = POLLIN;
945d2e
+    rc = poll(&pollfd, 1, timeout_ms);
945d2e
+    if (rc < 0) {
945d2e
+        return errno;
945d2e
+    } else if (rc == 0) {
945d2e
+        return EAGAIN;
945d2e
+    }
945d2e
+    return pcmk_rc_ok;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Dispatch available messages on an IPC connection (without main loop)
945d2e
+ *
945d2e
+ * \param[in]  api  IPC API connection
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ *
945d2e
+ * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
945d2e
+ *       this function when IPC data is available.
945d2e
+ */
945d2e
+void
945d2e
+pcmk_dispatch_ipc(pcmk_ipc_api_t *api)
945d2e
+{
945d2e
+    if (api == NULL) {
945d2e
+        return;
945d2e
+    }
945d2e
+    while (crm_ipc_ready(api->ipc)) {
945d2e
+        if (crm_ipc_read(api->ipc) > 0) {
945d2e
+            dispatch_ipc_data(crm_ipc_buffer(api->ipc), 0, api);
945d2e
+        }
945d2e
+    }
945d2e
+}
945d2e
+
945d2e
+// \return Standard Pacemaker return code
945d2e
+static int
945d2e
+connect_with_main_loop(pcmk_ipc_api_t *api)
945d2e
+{
945d2e
+    int rc;
945d2e
+
945d2e
+    struct ipc_client_callbacks callbacks = {
945d2e
+        .dispatch = dispatch_ipc_data,
945d2e
+        .destroy = ipc_post_disconnect,
945d2e
+    };
945d2e
+
945d2e
+    rc = pcmk__add_mainloop_ipc(api->ipc, G_PRIORITY_DEFAULT, api,
945d2e
+                                &callbacks, &(api->mainloop_io));
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
+        return rc;
945d2e
+    }
945d2e
+    crm_debug("Connected to %s IPC (attached to main loop)",
945d2e
+              pcmk_ipc_name(api, true));
945d2e
+    /* After this point, api->mainloop_io owns api->ipc, so api->ipc
945d2e
+     * should not be explicitly freed.
945d2e
+     */
945d2e
+    return pcmk_rc_ok;
945d2e
+}
945d2e
+
945d2e
+// \return Standard Pacemaker return code
945d2e
+static int
945d2e
+connect_without_main_loop(pcmk_ipc_api_t *api)
945d2e
+{
945d2e
+    int rc;
945d2e
+
945d2e
+    if (!crm_ipc_connect(api->ipc)) {
945d2e
+        rc = errno;
945d2e
+        crm_ipc_close(api->ipc);
945d2e
+        return rc;
945d2e
+    }
945d2e
+    crm_debug("Connected to %s IPC (without main loop)",
945d2e
+              pcmk_ipc_name(api, true));
945d2e
+    return pcmk_rc_ok;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Connect to a Pacemaker daemon via IPC
945d2e
+ *
945d2e
+ * \param[in]  api            IPC API instance
945d2e
+ * \param[out] dispatch_type  How IPC replies should be dispatched
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ */
945d2e
+int
945d2e
+pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type)
945d2e
+{
945d2e
+    int rc = pcmk_rc_ok;
945d2e
+
945d2e
+    if ((api == NULL) || (api->ipc == NULL)) {
945d2e
+        crm_err("Cannot connect to uninitialized API object");
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+
945d2e
+    if (crm_ipc_connected(api->ipc)) {
945d2e
+        crm_trace("Already connected to %s IPC API", pcmk_ipc_name(api, true));
945d2e
+        return pcmk_rc_ok;
945d2e
+    }
945d2e
+
945d2e
+    api->dispatch_type = dispatch_type;
945d2e
+    switch (dispatch_type) {
945d2e
+        case pcmk_ipc_dispatch_main:
945d2e
+            rc = connect_with_main_loop(api);
945d2e
+            break;
945d2e
+
945d2e
+        case pcmk_ipc_dispatch_sync:
945d2e
+        case pcmk_ipc_dispatch_poll:
945d2e
+            rc = connect_without_main_loop(api);
945d2e
+            break;
945d2e
+    }
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
+        return rc;
945d2e
+    }
945d2e
+
945d2e
+    if ((api->cmds != NULL) && (api->cmds->post_connect != NULL)) {
945d2e
+        rc = api->cmds->post_connect(api);
945d2e
+        if (rc != pcmk_rc_ok) {
945d2e
+            crm_ipc_close(api->ipc);
945d2e
+        }
945d2e
+    }
945d2e
+    return rc;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Disconnect an IPC API instance
945d2e
+ *
945d2e
+ * \param[in]  api  IPC API connection
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ *
945d2e
+ * \note If the connection is attached to a main loop, this function should be
945d2e
+ *       called before quitting the main loop, to ensure that all memory is
945d2e
+ *       freed.
945d2e
+ */
945d2e
+void
945d2e
+pcmk_disconnect_ipc(pcmk_ipc_api_t *api)
945d2e
+{
945d2e
+    if ((api == NULL) || (api->ipc == NULL)) {
945d2e
+        return;
945d2e
+    }
945d2e
+    switch (api->dispatch_type) {
945d2e
+        case pcmk_ipc_dispatch_main:
945d2e
+            {
945d2e
+                mainloop_io_t *mainloop_io = api->mainloop_io;
945d2e
+
945d2e
+                // Make sure no code with access to api can use these again
945d2e
+                api->mainloop_io = NULL;
945d2e
+                api->ipc = NULL;
945d2e
+
945d2e
+                mainloop_del_ipc_client(mainloop_io);
945d2e
+                // After this point api might have already been freed
945d2e
+            }
945d2e
+            break;
945d2e
+
945d2e
+        case pcmk_ipc_dispatch_poll:
945d2e
+        case pcmk_ipc_dispatch_sync:
945d2e
+            {
945d2e
+                crm_ipc_t *ipc = api->ipc;
945d2e
+
945d2e
+                // Make sure no code with access to api can use ipc again
945d2e
+                api->ipc = NULL;
945d2e
+
945d2e
+                // This should always be the case already, but to be safe
945d2e
+                api->free_on_disconnect = false;
945d2e
+
945d2e
+                crm_ipc_destroy(ipc);
945d2e
+                ipc_post_disconnect(api);
945d2e
+            }
945d2e
+            break;
945d2e
+    }
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Register a callback for IPC API events
945d2e
+ *
945d2e
+ * \param[in] api          IPC API connection
945d2e
+ * \param[in] callback     Callback to register
945d2e
+ * \param[in] userdata     Caller data to pass to callback
945d2e
+ *
945d2e
+ * \note This function may be called multiple times to update the callback
945d2e
+ *       and/or user data. The caller remains responsible for freeing
945d2e
+ *       userdata in any case (after the IPC is disconnected, if the
945d2e
+ *       user data is still registered with the IPC).
945d2e
+ */
945d2e
+void
945d2e
+pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb,
945d2e
+                           void *user_data)
945d2e
+{
945d2e
+    if (api == NULL) {
945d2e
+        return;
945d2e
+    }
945d2e
+    api->cb = cb;
945d2e
+    api->user_data = user_data;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \internal
945d2e
+ * \brief Send an XML request across an IPC API connection
945d2e
+ *
945d2e
+ * \param[in] api          IPC API connection
945d2e
+ * \param[in] request      XML request to send
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ *
945d2e
+ * \note Daemon-specific IPC API functions should call this function to send
945d2e
+ *       requests, because it handles different dispatch types appropriately.
945d2e
+ */
945d2e
+int
945d2e
+pcmk__send_ipc_request(pcmk_ipc_api_t *api, xmlNode *request)
945d2e
+{
945d2e
+    int rc;
945d2e
+    xmlNode *reply = NULL;
945d2e
+    enum crm_ipc_flags flags = crm_ipc_flags_none;
945d2e
+
945d2e
+    if ((api == NULL) || (api->ipc == NULL) || (request == NULL)) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+    crm_log_xml_trace(request, "ipc-sent");
945d2e
+
945d2e
+    // Synchronous dispatch requires waiting for a reply
945d2e
+    if ((api->dispatch_type == pcmk_ipc_dispatch_sync)
945d2e
+        && (api->cmds != NULL)
945d2e
+        && (api->cmds->reply_expected != NULL)
945d2e
+        && (api->cmds->reply_expected(api, request))) {
945d2e
+        flags = crm_ipc_client_response;
945d2e
+    }
945d2e
+
945d2e
+    // The 0 here means a default timeout of 5 seconds
945d2e
+    rc = crm_ipc_send(api->ipc, request, flags, 0, &reply);
945d2e
+
945d2e
+    if (rc < 0) {
945d2e
+        return pcmk_legacy2rc(rc);
945d2e
+    } else if (rc == 0) {
945d2e
+        return ENODATA;
945d2e
+    }
945d2e
+
945d2e
+    // With synchronous dispatch, we dispatch any reply now
945d2e
+    if (reply != NULL) {
945d2e
+        call_api_dispatch(api, reply);
945d2e
+        free_xml(reply);
945d2e
+    }
945d2e
+    return pcmk_rc_ok;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \internal
945d2e
+ * \brief Create the XML for an IPC request to purge a node from the peer cache
945d2e
+ *
945d2e
+ * \param[in]  api        IPC API connection
945d2e
+ * \param[in]  node_name  If not NULL, name of node to purge
945d2e
+ * \param[in]  nodeid     If not 0, node ID of node to purge
945d2e
+ *
945d2e
+ * \return Newly allocated IPC request XML
945d2e
+ *
945d2e
+ * \note The controller, fencer, and pacemakerd use the same request syntax, but
945d2e
+ *       the attribute manager uses a different one. The CIB manager doesn't
945d2e
+ *       have any syntax for it. The executor and scheduler don't connect to the
945d2e
+ *       cluster layer and thus don't have or need any syntax for it.
945d2e
+ *
945d2e
+ * \todo Modify the attribute manager to accept the common syntax (as well
945d2e
+ *       as its current one, for compatibility with older clients). Modify
945d2e
+ *       the CIB manager to accept and honor the common syntax. Modify the
945d2e
+ *       executor and scheduler to accept the syntax (immediately returning
945d2e
+ *       success), just for consistency. Modify this function to use the
945d2e
+ *       common syntax with all daemons if their version supports it.
945d2e
+ */
945d2e
+static xmlNode *
945d2e
+create_purge_node_request(pcmk_ipc_api_t *api, const char *node_name,
945d2e
+                          uint32_t nodeid)
945d2e
+{
945d2e
+    xmlNode *request = NULL;
945d2e
+    const char *client = crm_system_name? crm_system_name : "client";
945d2e
+
945d2e
+    switch (api->server) {
945d2e
+        case pcmk_ipc_attrd:
945d2e
+            request = create_xml_node(NULL, __FUNCTION__);
945d2e
+            crm_xml_add(request, F_TYPE, T_ATTRD);
945d2e
+            crm_xml_add(request, F_ORIG, crm_system_name);
945d2e
+            crm_xml_add(request, PCMK__XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE);
945d2e
+            crm_xml_add(request, PCMK__XA_ATTR_NODE_NAME, node_name);
945d2e
+            if (nodeid > 0) {
945d2e
+                crm_xml_add_int(request, PCMK__XA_ATTR_NODE_ID, (int) nodeid);
945d2e
+            }
945d2e
+            break;
945d2e
+
945d2e
+        case pcmk_ipc_controld:
945d2e
+        case pcmk_ipc_fenced:
945d2e
+        case pcmk_ipc_pacemakerd:
945d2e
+            request = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL,
945d2e
+                                     pcmk_ipc_name(api, false), client, NULL);
945d2e
+            if (nodeid > 0) {
945d2e
+                crm_xml_set_id(request, "%lu", (unsigned long) nodeid);
945d2e
+            }
945d2e
+            crm_xml_add(request, XML_ATTR_UNAME, node_name);
945d2e
+            break;
945d2e
+
945d2e
+        case pcmk_ipc_based:
945d2e
+        case pcmk_ipc_execd:
945d2e
+        case pcmk_ipc_schedulerd:
945d2e
+            break;
945d2e
+    }
945d2e
+    return request;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Ask a Pacemaker daemon to purge a node from its peer cache
945d2e
+ *
945d2e
+ * \param[in]  api        IPC API connection
945d2e
+ * \param[in]  node_name  If not NULL, name of node to purge
945d2e
+ * \param[in]  nodeid     If not 0, node ID of node to purge
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ *
945d2e
+ * \note At least one of node_name or nodeid must be specified.
945d2e
+ */
945d2e
+int
945d2e
+pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name, uint32_t nodeid)
945d2e
+{
945d2e
+    int rc = 0;
945d2e
+    xmlNode *request = NULL;
945d2e
+
945d2e
+    if (api == NULL) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+    if ((node_name == NULL) && (nodeid == 0)) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+
945d2e
+    request = create_purge_node_request(api, node_name, nodeid);
945d2e
+    if (request == NULL) {
945d2e
+        return EOPNOTSUPP;
945d2e
+    }
945d2e
+    rc = pcmk__send_ipc_request(api, request);
945d2e
+    free_xml(request);
945d2e
+
945d2e
+    crm_debug("%s peer cache purge of node %s[%lu]: rc=%d",
945d2e
+              pcmk_ipc_name(api, true), node_name, (unsigned long) nodeid, rc);
945d2e
+    return rc;
945d2e
+}
945d2e
+
945d2e
+/*
945d2e
+ * Generic IPC API (to eventually be deprecated as public API and made internal)
945d2e
+ */
945d2e
+
945d2e
 struct crm_ipc_s {
945d2e
     struct pollfd pfd;
945d2e
     unsigned int max_buf_size; // maximum bytes we can send or receive over IPC
945d2e
@@ -42,16 +715,44 @@ struct crm_ipc_s {
945d2e
     qb_ipcc_connection_t *ipc;
945d2e
 };
945d2e
 
945d2e
+/*!
945d2e
+ * \brief Create a new (legacy) object for using Pacemaker daemon IPC
945d2e
+ *
945d2e
+ * \param[in] name      IPC system name to connect to
945d2e
+ * \param[in] max_size  Use a maximum IPC buffer size of at least this size
945d2e
+ *
945d2e
+ * \return Newly allocated IPC object on success, NULL otherwise
945d2e
+ *
945d2e
+ * \note The caller is responsible for freeing the result using
945d2e
+ *       crm_ipc_destroy().
945d2e
+ * \note This should be considered deprecated for use with daemons supported by
945d2e
+ *       pcmk_new_ipc_api().
945d2e
+ */
945d2e
 crm_ipc_t *
945d2e
 crm_ipc_new(const char *name, size_t max_size)
945d2e
 {
945d2e
     crm_ipc_t *client = NULL;
945d2e
 
945d2e
     client = calloc(1, sizeof(crm_ipc_t));
945d2e
+    if (client == NULL) {
945d2e
+        crm_err("Could not create IPC connection: %s", strerror(errno));
945d2e
+        return NULL;
945d2e
+    }
945d2e
 
945d2e
     client->name = strdup(name);
945d2e
+    if (client->name == NULL) {
945d2e
+        crm_err("Could not create IPC connection: %s", strerror(errno));
945d2e
+        free(client);
945d2e
+        return NULL;
945d2e
+    }
945d2e
     client->buf_size = pcmk__ipc_buffer_size(max_size);
945d2e
     client->buffer = malloc(client->buf_size);
945d2e
+    if (client->buffer == NULL) {
945d2e
+        crm_err("Could not create IPC connection: %s", strerror(errno));
945d2e
+        free(client->name);
945d2e
+        free(client);
945d2e
+        return NULL;
945d2e
+    }
945d2e
 
945d2e
     /* Clients initiating connection pick the max buf size */
945d2e
     client->max_buf_size = client->buf_size;
945d2e
@@ -143,8 +844,6 @@ void
945d2e
 crm_ipc_close(crm_ipc_t * client)
945d2e
 {
945d2e
     if (client) {
945d2e
-        crm_trace("Disconnecting %s IPC connection %p (%p)", client->name, client, client->ipc);
945d2e
-
945d2e
         if (client->ipc) {
945d2e
             qb_ipcc_connection_t *ipc = client->ipc;
945d2e
 
945d2e
diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c
945d2e
index 634eead..e942e57 100644
945d2e
--- a/lib/common/mainloop.c
945d2e
+++ b/lib/common/mainloop.c
945d2e
@@ -834,32 +834,66 @@ mainloop_gio_destroy(gpointer c)
945d2e
     free(c_name);
945d2e
 }
945d2e
 
945d2e
-mainloop_io_t *
945d2e
-mainloop_add_ipc_client(const char *name, int priority, size_t max_size, void *userdata,
945d2e
-                        struct ipc_client_callbacks *callbacks)
945d2e
+/*!
945d2e
+ * \brief Connect to IPC and add it as a main loop source
945d2e
+ *
945d2e
+ * \param[in]  ipc        IPC connection to add
945d2e
+ * \param[in]  priority   Event source priority to use for connection
945d2e
+ * \param[in]  userdata   Data to register with callbacks
945d2e
+ * \param[in]  callbacks  Dispatch and destroy callbacks for connection
945d2e
+ * \param[out] source     Newly allocated event source
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ *
945d2e
+ * \note On failure, the caller is still responsible for ipc. On success, the
945d2e
+ *       caller should call mainloop_del_ipc_client() when source is no longer
945d2e
+ *       needed, which will lead to the disconnection of the IPC later in the
945d2e
+ *       main loop if it is connected. However the IPC disconnects,
945d2e
+ *       mainloop_gio_destroy() will free ipc and source after calling the
945d2e
+ *       destroy callback.
945d2e
+ */
945d2e
+int
945d2e
+pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata,
945d2e
+                       struct ipc_client_callbacks *callbacks,
945d2e
+                       mainloop_io_t **source)
945d2e
 {
945d2e
-    mainloop_io_t *client = NULL;
945d2e
-    crm_ipc_t *conn = crm_ipc_new(name, max_size);
945d2e
+    CRM_CHECK((ipc != NULL) && (callbacks != NULL), return EINVAL);
945d2e
 
945d2e
-    if (conn && crm_ipc_connect(conn)) {
945d2e
-        int32_t fd = crm_ipc_get_fd(conn);
945d2e
+    if (!crm_ipc_connect(ipc)) {
945d2e
+        return ENOTCONN;
945d2e
+    }
945d2e
+    *source = mainloop_add_fd(crm_ipc_name(ipc), priority, crm_ipc_get_fd(ipc),
945d2e
+                              userdata, NULL);
945d2e
+    if (*source == NULL) {
945d2e
+        int rc = errno;
945d2e
 
945d2e
-        client = mainloop_add_fd(name, priority, fd, userdata, NULL);
945d2e
+        crm_ipc_close(ipc);
945d2e
+        return rc;
945d2e
     }
945d2e
+    (*source)->ipc = ipc;
945d2e
+    (*source)->destroy_fn = callbacks->destroy;
945d2e
+    (*source)->dispatch_fn_ipc = callbacks->dispatch;
945d2e
+    return pcmk_rc_ok;
945d2e
+}
945d2e
 
945d2e
-    if (client == NULL) {
945d2e
-        crm_perror(LOG_TRACE, "Connection to %s failed", name);
945d2e
-        if (conn) {
945d2e
-            crm_ipc_close(conn);
945d2e
-            crm_ipc_destroy(conn);
945d2e
+mainloop_io_t *
945d2e
+mainloop_add_ipc_client(const char *name, int priority, size_t max_size,
945d2e
+                        void *userdata, struct ipc_client_callbacks *callbacks)
945d2e
+{
945d2e
+    crm_ipc_t *ipc = crm_ipc_new(name, max_size);
945d2e
+    mainloop_io_t *source = NULL;
945d2e
+    int rc = pcmk__add_mainloop_ipc(ipc, priority, userdata, callbacks,
945d2e
+                                    &source);
945d2e
+
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
+        if (crm_log_level == LOG_STDOUT) {
945d2e
+            fprintf(stderr, "Connection to %s failed: %s",
945d2e
+                    name, pcmk_rc_str(rc));
945d2e
         }
945d2e
+        crm_ipc_destroy(ipc);
945d2e
         return NULL;
945d2e
     }
945d2e
-
945d2e
-    client->ipc = conn;
945d2e
-    client->destroy_fn = callbacks->destroy;
945d2e
-    client->dispatch_fn_ipc = callbacks->dispatch;
945d2e
-    return client;
945d2e
+    return source;
945d2e
 }
945d2e
 
945d2e
 void
945d2e
-- 
945d2e
1.8.3.1
945d2e
945d2e
945d2e
From b9539da27998ff5e6c8b681f39603550a923ca33 Mon Sep 17 00:00:00 2001
945d2e
From: Ken Gaillot <kgaillot@redhat.com>
945d2e
Date: Mon, 6 Apr 2020 17:40:55 -0500
945d2e
Subject: [PATCH 2/6] Refactor: libcrmcommon: add C API for controller IPC
945d2e
945d2e
Implement a C API for controller IPC using the new IPC API model.
945d2e
---
945d2e
 include/crm/common/Makefile.am    |   2 +-
945d2e
 include/crm/common/ipc.h          |   4 +-
945d2e
 include/crm/common/ipc_controld.h |  99 +++++++
945d2e
 lib/common/Makefile.am            |   1 +
945d2e
 lib/common/crmcommon_private.h    |   6 +
945d2e
 lib/common/ipc_client.c           |  46 +--
945d2e
 lib/common/ipc_controld.c         | 609 ++++++++++++++++++++++++++++++++++++++
945d2e
 7 files changed, 723 insertions(+), 44 deletions(-)
945d2e
 create mode 100644 include/crm/common/ipc_controld.h
945d2e
 create mode 100644 lib/common/ipc_controld.c
945d2e
945d2e
diff --git a/include/crm/common/Makefile.am b/include/crm/common/Makefile.am
945d2e
index 776e4a7..f29d105 100644
945d2e
--- a/include/crm/common/Makefile.am
945d2e
+++ b/include/crm/common/Makefile.am
945d2e
@@ -12,7 +12,7 @@ MAINTAINERCLEANFILES = Makefile.in
945d2e
 headerdir=$(pkgincludedir)/crm/common
945d2e
 
945d2e
 header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h results.h \
945d2e
-		 nvpair.h acl.h
945d2e
+		 nvpair.h acl.h ipc_controld.h
945d2e
 noinst_HEADERS = internal.h alerts_internal.h \
945d2e
 		 iso8601_internal.h remote_internal.h xml_internal.h \
945d2e
 		 ipc_internal.h output.h cmdline_internal.h curses_internal.h \
945d2e
diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h
945d2e
index 8dee1b1..c67aaea 100644
945d2e
--- a/include/crm/common/ipc.h
945d2e
+++ b/include/crm/common/ipc.h
945d2e
@@ -217,7 +217,9 @@ unsigned int crm_ipc_default_buffer_size(void);
945d2e
 int crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid,
945d2e
                                  pid_t *gotpid, uid_t *gotuid, gid_t *gotgid);
945d2e
 
945d2e
-/* Utils */
945d2e
+/* This is controller-specific but is declared in this header for C API
945d2e
+ * backward compatibility.
945d2e
+ */
945d2e
 xmlNode *create_hello_message(const char *uuid, const char *client_name,
945d2e
                               const char *major_version, const char *minor_version);
945d2e
 
945d2e
diff --git a/include/crm/common/ipc_controld.h b/include/crm/common/ipc_controld.h
945d2e
new file mode 100644
945d2e
index 0000000..0ebabfc
945d2e
--- /dev/null
945d2e
+++ b/include/crm/common/ipc_controld.h
945d2e
@@ -0,0 +1,99 @@
945d2e
+/*
945d2e
+ * Copyright 2020 the Pacemaker project contributors
945d2e
+ *
945d2e
+ * The version control history for this file may have further details.
945d2e
+ *
945d2e
+ * This source code is licensed under the GNU Lesser General Public License
945d2e
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
945d2e
+ */
945d2e
+
945d2e
+#ifndef PCMK__IPC_CONTROLD__H
945d2e
+#  define PCMK__IPC_CONTROLD__H
945d2e
+
945d2e
+#ifdef __cplusplus
945d2e
+extern "C" {
945d2e
+#endif
945d2e
+
945d2e
+/**
945d2e
+ * \file
945d2e
+ * \brief IPC commands for Pacemaker controller
945d2e
+ *
945d2e
+ * \ingroup core
945d2e
+ */
945d2e
+
945d2e
+#include <stdbool.h>                    // bool
945d2e
+#include <libxml/tree.h>                // xmlNode
945d2e
+#include <crm/common/ipc.h>             // pcmk_ipc_api_t
945d2e
+
945d2e
+//! Possible types of controller replies
945d2e
+enum pcmk_controld_api_reply {
945d2e
+    pcmk_controld_reply_unknown,
945d2e
+    pcmk_controld_reply_reprobe,
945d2e
+    pcmk_controld_reply_info,
945d2e
+    pcmk_controld_reply_resource,
945d2e
+    pcmk_controld_reply_ping,
945d2e
+};
945d2e
+
945d2e
+/*!
945d2e
+ * Controller reply passed to event callback
945d2e
+ *
945d2e
+ * \note Shutdown and election calls have no reply. Reprobe calls are
945d2e
+ *       acknowledged but contain no data (reply_type will be the only item
945d2e
+ *       set). Node info and ping calls have their own reply data. Fail and
945d2e
+ *       refresh calls use the resource reply type and reply data.
945d2e
+ * \note The pointers in the reply are only guaranteed to be meaningful for the
945d2e
+ *       execution of the callback; if the values are needed for later, the
945d2e
+ *       callback should copy them.
945d2e
+ */
945d2e
+typedef struct {
945d2e
+    enum pcmk_controld_api_reply reply_type;
945d2e
+    const char *feature_set; //!< CRM feature set advertised by controller
945d2e
+    const char *host_from;   //!< Name of node that sent reply
945d2e
+
945d2e
+    union {
945d2e
+        // pcmk_controld_reply_info
945d2e
+        struct {
945d2e
+            bool have_quorum;
945d2e
+            bool is_remote;
945d2e
+            int id;
945d2e
+            const char *uuid;
945d2e
+            const char *uname;
945d2e
+            const char *state;
945d2e
+        } node_info;
945d2e
+
945d2e
+        // pcmk_controld_reply_resource
945d2e
+        struct {
945d2e
+            xmlNode *node_state;    //
945d2e
+        } resource;
945d2e
+
945d2e
+        // pcmk_controld_reply_ping
945d2e
+        struct {
945d2e
+            const char *sys_from;
945d2e
+            const char *fsa_state;
945d2e
+            const char *result;
945d2e
+        } ping;
945d2e
+    } data;
945d2e
+} pcmk_controld_api_reply_t;
945d2e
+
945d2e
+int pcmk_controld_api_reprobe(pcmk_ipc_api_t *api, const char *target_node,
945d2e
+                              const char *router_node);
945d2e
+int pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid);
945d2e
+int pcmk_controld_api_fail(pcmk_ipc_api_t *api, const char *target_node,
945d2e
+                           const char *router_node, const char *rsc_id,
945d2e
+                           const char *rsc_long_id, const char *standard,
945d2e
+                           const char *provider, const char *type);
945d2e
+int pcmk_controld_api_refresh(pcmk_ipc_api_t *api, const char *target_node,
945d2e
+                              const char *router_node, const char *rsc_id,
945d2e
+                              const char *rsc_long_id, const char *standard,
945d2e
+                              const char *provider, const char *type,
945d2e
+                              bool cib_only);
945d2e
+int pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name);
945d2e
+int pcmk_controld_api_shutdown(pcmk_ipc_api_t *api, const char *node_name);
945d2e
+int pcmk_controld_api_start_election(pcmk_ipc_api_t *api);
945d2e
+unsigned int pcmk_controld_api_replies_expected(pcmk_ipc_api_t *api);
945d2e
+
945d2e
+#ifdef __cplusplus
945d2e
+}
945d2e
+#endif
945d2e
+
945d2e
+#endif // PCMK__IPC_CONTROLD__H
945d2e
diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am
945d2e
index 29404a6..db66a6e 100644
945d2e
--- a/lib/common/Makefile.am
945d2e
+++ b/lib/common/Makefile.am
945d2e
@@ -49,6 +49,7 @@ libcrmcommon_la_SOURCES	+= digest.c
945d2e
 libcrmcommon_la_SOURCES	+= io.c
945d2e
 libcrmcommon_la_SOURCES	+= ipc_client.c
945d2e
 libcrmcommon_la_SOURCES	+= ipc_common.c
945d2e
+libcrmcommon_la_SOURCES	+= ipc_controld.c
945d2e
 libcrmcommon_la_SOURCES	+= ipc_server.c
945d2e
 libcrmcommon_la_SOURCES	+= iso8601.c
945d2e
 libcrmcommon_la_SOURCES	+= logging.c
945d2e
diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h
945d2e
index f9df27d..49dae6c 100644
945d2e
--- a/lib/common/crmcommon_private.h
945d2e
+++ b/lib/common/crmcommon_private.h
945d2e
@@ -103,6 +103,9 @@ pcmk__xml_attr_value(const xmlAttr *attr)
945d2e
 
945d2e
 #define PCMK__IPC_VERSION 1
945d2e
 
945d2e
+#define PCMK__CONTROLD_API_MAJOR "1"
945d2e
+#define PCMK__CONTROLD_API_MINOR "0"
945d2e
+
945d2e
 // IPC behavior that varies by daemon
945d2e
 typedef struct pcmk__ipc_methods_s {
945d2e
     /*!
945d2e
@@ -203,4 +206,7 @@ unsigned int pcmk__ipc_buffer_size(unsigned int max);
945d2e
 G_GNUC_INTERNAL
945d2e
 bool pcmk__valid_ipc_header(const pcmk__ipc_header_t *header);
945d2e
 
945d2e
+G_GNUC_INTERNAL
945d2e
+pcmk__ipc_methods_t *pcmk__controld_api_methods(void);
945d2e
+
945d2e
 #endif  // CRMCOMMON_PRIVATE__H
945d2e
diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c
945d2e
index 16dc9b5..4077d61 100644
945d2e
--- a/lib/common/ipc_client.c
945d2e
+++ b/lib/common/ipc_client.c
945d2e
@@ -40,7 +40,8 @@
945d2e
  * \return Standard Pacemaker result code
945d2e
  *
945d2e
  * \note The caller is responsible for freeing *api using pcmk_free_ipc_api().
945d2e
- * \note This is intended to supersede crm_ipc_new() but is not yet usable.
945d2e
+ * \note This is intended to supersede crm_ipc_new() but currently only
945d2e
+ *       supports the controller IPC API.
945d2e
  */
945d2e
 int
945d2e
 pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
945d2e
@@ -73,6 +74,7 @@ pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
945d2e
             break;
945d2e
 
945d2e
         case pcmk_ipc_controld:
945d2e
+            (*api)->cmds = pcmk__controld_api_methods();
945d2e
             break;
945d2e
 
945d2e
         case pcmk_ipc_execd:
945d2e
@@ -247,7 +249,7 @@ pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log)
945d2e
             return for_log? "CIB manager" : NULL /* PCMK__SERVER_BASED_RW */;
945d2e
 
945d2e
         case pcmk_ipc_controld:
945d2e
-            return for_log? "controller" : NULL /* CRM_SYSTEM_CRMD */;
945d2e
+            return for_log? "controller" : CRM_SYSTEM_CRMD;
945d2e
 
945d2e
         case pcmk_ipc_execd:
945d2e
             return for_log? "executor" : NULL /* CRM_SYSTEM_LRMD */;
945d2e
@@ -1412,43 +1414,3 @@ bail:
945d2e
     }
945d2e
     return rc;
945d2e
 }
945d2e
-
945d2e
-xmlNode *
945d2e
-create_hello_message(const char *uuid,
945d2e
-                     const char *client_name, const char *major_version, const char *minor_version)
945d2e
-{
945d2e
-    xmlNode *hello_node = NULL;
945d2e
-    xmlNode *hello = NULL;
945d2e
-
945d2e
-    if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name)
945d2e
-        || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) {
945d2e
-        crm_err("Could not create IPC hello message from %s (UUID %s): "
945d2e
-                "missing information",
945d2e
-                client_name? client_name : "unknown client",
945d2e
-                uuid? uuid : "unknown");
945d2e
-        return NULL;
945d2e
-    }
945d2e
-
945d2e
-    hello_node = create_xml_node(NULL, XML_TAG_OPTIONS);
945d2e
-    if (hello_node == NULL) {
945d2e
-        crm_err("Could not create IPC hello message from %s (UUID %s): "
945d2e
-                "Message data creation failed", client_name, uuid);
945d2e
-        return NULL;
945d2e
-    }
945d2e
-
945d2e
-    crm_xml_add(hello_node, "major_version", major_version);
945d2e
-    crm_xml_add(hello_node, "minor_version", minor_version);
945d2e
-    crm_xml_add(hello_node, "client_name", client_name);
945d2e
-    crm_xml_add(hello_node, "client_uuid", uuid);
945d2e
-
945d2e
-    hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
945d2e
-    if (hello == NULL) {
945d2e
-        crm_err("Could not create IPC hello message from %s (UUID %s): "
945d2e
-                "Request creation failed", client_name, uuid);
945d2e
-        return NULL;
945d2e
-    }
945d2e
-    free_xml(hello_node);
945d2e
-
945d2e
-    crm_trace("Created hello message from %s (UUID %s)", client_name, uuid);
945d2e
-    return hello;
945d2e
-}
945d2e
diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c
945d2e
new file mode 100644
945d2e
index 0000000..22bb733
945d2e
--- /dev/null
945d2e
+++ b/lib/common/ipc_controld.c
945d2e
@@ -0,0 +1,609 @@
945d2e
+/*
945d2e
+ * Copyright 2020 the Pacemaker project contributors
945d2e
+ *
945d2e
+ * The version control history for this file may have further details.
945d2e
+ *
945d2e
+ * This source code is licensed under the GNU Lesser General Public License
945d2e
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
945d2e
+ */
945d2e
+
945d2e
+#include <crm_internal.h>
945d2e
+
945d2e
+#include <stdio.h>
945d2e
+#include <stdbool.h>
945d2e
+#include <errno.h>
945d2e
+#include <libxml/tree.h>
945d2e
+
945d2e
+#include <crm/crm.h>
945d2e
+#include <crm/msg_xml.h>
945d2e
+#include <crm/common/xml.h>
945d2e
+#include <crm/common/ipc.h>
945d2e
+#include <crm/common/ipc_internal.h>
945d2e
+#include <crm/common/ipc_controld.h>
945d2e
+#include "crmcommon_private.h"
945d2e
+
945d2e
+struct controld_api_private_s {
945d2e
+    char *client_uuid;
945d2e
+    unsigned int replies_expected;
945d2e
+};
945d2e
+
945d2e
+// \return Standard Pacemaker return code
945d2e
+static int
945d2e
+new_data(pcmk_ipc_api_t *api)
945d2e
+{
945d2e
+    struct controld_api_private_s *private = NULL;
945d2e
+
945d2e
+    api->api_data = calloc(1, sizeof(struct controld_api_private_s));
945d2e
+
945d2e
+    if (api->api_data == NULL) {
945d2e
+        return errno;
945d2e
+    }
945d2e
+
945d2e
+    private = api->api_data;
945d2e
+
945d2e
+    /* This is set to the PID because that's how it was always done, but PIDs
945d2e
+     * are not unique because clients can be remote. The value appears to be
945d2e
+     * unused other than as part of F_CRM_SYS_FROM in IPC requests, which is
945d2e
+     * only compared against the internal system names (CRM_SYSTEM_TENGINE,
945d2e
+     * etc.), so it shouldn't be a problem.
945d2e
+     */
945d2e
+    private->client_uuid = pcmk__getpid_s();
945d2e
+
945d2e
+    /* @TODO Implement a call ID model similar to the CIB, executor, and fencer
945d2e
+     *       IPC APIs, so that requests and replies can be matched, and
945d2e
+     *       duplicate replies can be discarded.
945d2e
+     */
945d2e
+    return pcmk_rc_ok;
945d2e
+}
945d2e
+
945d2e
+static void
945d2e
+free_data(void *data)
945d2e
+{
945d2e
+    free(((struct controld_api_private_s *) data)->client_uuid);
945d2e
+    free(data);
945d2e
+}
945d2e
+
945d2e
+// \return Standard Pacemaker return code
945d2e
+static int
945d2e
+post_connect(pcmk_ipc_api_t *api)
945d2e
+{
945d2e
+    /* The controller currently requires clients to register via a hello
945d2e
+     * request, but does not reply back.
945d2e
+     */
945d2e
+    struct controld_api_private_s *private = api->api_data;
945d2e
+    const char *client_name = crm_system_name? crm_system_name : "client";
945d2e
+    xmlNode *hello;
945d2e
+    int rc;
945d2e
+
945d2e
+    hello = create_hello_message(private->client_uuid, client_name,
945d2e
+                                 PCMK__CONTROLD_API_MAJOR,
945d2e
+                                 PCMK__CONTROLD_API_MINOR);
945d2e
+    rc = pcmk__send_ipc_request(api, hello);
945d2e
+    free_xml(hello);
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
+        crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s",
945d2e
+                 pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc);
945d2e
+    } else {
945d2e
+        crm_debug("Sent IPC hello to %s", pcmk_ipc_name(api, true));
945d2e
+    }
945d2e
+    return rc;
945d2e
+}
945d2e
+
945d2e
+#define xml_true(xml, field) crm_is_true(crm_element_value(xml, field))
945d2e
+
945d2e
+static void
945d2e
+set_node_info_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
945d2e
+{
945d2e
+    data->reply_type = pcmk_controld_reply_info;
945d2e
+    if (msg_data == NULL) {
945d2e
+        return;
945d2e
+    }
945d2e
+    data->data.node_info.have_quorum = xml_true(msg_data, XML_ATTR_HAVE_QUORUM);
945d2e
+    data->data.node_info.is_remote = xml_true(msg_data, XML_NODE_IS_REMOTE);
945d2e
+    crm_element_value_int(msg_data, XML_ATTR_ID, &(data->data.node_info.id));
945d2e
+    data->data.node_info.uuid = crm_element_value(msg_data, XML_ATTR_UUID);
945d2e
+    data->data.node_info.uname = crm_element_value(msg_data, XML_ATTR_UNAME);
945d2e
+    data->data.node_info.state = crm_element_value(msg_data, XML_NODE_IS_PEER);
945d2e
+}
945d2e
+
945d2e
+static void
945d2e
+set_ping_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
945d2e
+{
945d2e
+    data->reply_type = pcmk_controld_reply_ping;
945d2e
+    if (msg_data == NULL) {
945d2e
+        return;
945d2e
+    }
945d2e
+    data->data.ping.sys_from = crm_element_value(msg_data,
945d2e
+                                                 XML_PING_ATTR_SYSFROM);
945d2e
+    data->data.ping.fsa_state = crm_element_value(msg_data,
945d2e
+                                                  XML_PING_ATTR_CRMDSTATE);
945d2e
+    data->data.ping.result = crm_element_value(msg_data, XML_PING_ATTR_STATUS);
945d2e
+}
945d2e
+
945d2e
+static bool
945d2e
+reply_expected(pcmk_ipc_api_t *api, xmlNode *request)
945d2e
+{
945d2e
+    const char *command = crm_element_value(request, F_CRM_TASK);
945d2e
+
945d2e
+    if (command == NULL) {
945d2e
+        return false;
945d2e
+    }
945d2e
+
945d2e
+    // We only need to handle commands that functions in this file can send
945d2e
+    return !strcmp(command, CRM_OP_REPROBE)
945d2e
+           || !strcmp(command, CRM_OP_NODE_INFO)
945d2e
+           || !strcmp(command, CRM_OP_PING)
945d2e
+           || !strcmp(command, CRM_OP_LRM_FAIL)
945d2e
+           || !strcmp(command, CRM_OP_LRM_DELETE);
945d2e
+}
945d2e
+
945d2e
+static void
945d2e
+dispatch(pcmk_ipc_api_t *api, xmlNode *reply)
945d2e
+{
945d2e
+    struct controld_api_private_s *private = api->api_data;
945d2e
+    crm_exit_t status = CRM_EX_OK;
945d2e
+    xmlNode *msg_data = NULL;
945d2e
+    const char *value = NULL;
945d2e
+    pcmk_controld_api_reply_t reply_data = {
945d2e
+        pcmk_controld_reply_unknown, NULL, NULL,
945d2e
+    };
945d2e
+
945d2e
+    if (private->replies_expected > 0) {
945d2e
+        private->replies_expected--;
945d2e
+    }
945d2e
+
945d2e
+    // Do some basic validation of the reply
945d2e
+
945d2e
+    /* @TODO We should be able to verify that value is always a response, but
945d2e
+     *       currently the controller doesn't always properly set the type. Even
945d2e
+     *       if we fix the controller, we'll still need to handle replies from
945d2e
+     *       old versions (feature set could be used to differentiate).
945d2e
+     */
945d2e
+    value = crm_element_value(reply, F_CRM_MSG_TYPE);
945d2e
+    if ((value == NULL) || (strcmp(value, XML_ATTR_REQUEST)
945d2e
+                            && strcmp(value, XML_ATTR_RESPONSE))) {
945d2e
+        crm_debug("Unrecognizable controller message: invalid message type '%s'",
945d2e
+                  crm_str(value));
945d2e
+        status = CRM_EX_PROTOCOL;
945d2e
+        reply = NULL;
945d2e
+    }
945d2e
+
945d2e
+    if (crm_element_value(reply, XML_ATTR_REFERENCE) == NULL) {
945d2e
+        crm_debug("Unrecognizable controller message: no reference");
945d2e
+        status = CRM_EX_PROTOCOL;
945d2e
+        reply = NULL;
945d2e
+    }
945d2e
+
945d2e
+    value = crm_element_value(reply, F_CRM_TASK);
945d2e
+    if (value == NULL) {
945d2e
+        crm_debug("Unrecognizable controller message: no command name");
945d2e
+        status = CRM_EX_PROTOCOL;
945d2e
+        reply = NULL;
945d2e
+    }
945d2e
+
945d2e
+    // Parse useful info from reply
945d2e
+
945d2e
+    if (reply != NULL) {
945d2e
+        reply_data.feature_set = crm_element_value(reply, XML_ATTR_VERSION);
945d2e
+        reply_data.host_from = crm_element_value(reply, F_CRM_HOST_FROM);
945d2e
+        msg_data = get_message_xml(reply, F_CRM_DATA);
945d2e
+
945d2e
+        if (!strcmp(value, CRM_OP_REPROBE)) {
945d2e
+            reply_data.reply_type = pcmk_controld_reply_reprobe;
945d2e
+
945d2e
+        } else if (!strcmp(value, CRM_OP_NODE_INFO)) {
945d2e
+            set_node_info_data(&reply_data, msg_data);
945d2e
+
945d2e
+        } else if (!strcmp(value, CRM_OP_INVOKE_LRM)) {
945d2e
+            reply_data.reply_type = pcmk_controld_reply_resource;
945d2e
+            reply_data.data.resource.node_state = msg_data;
945d2e
+
945d2e
+        } else if (!strcmp(value, CRM_OP_PING)) {
945d2e
+            set_ping_data(&reply_data, msg_data);
945d2e
+
945d2e
+        } else {
945d2e
+            crm_debug("Unrecognizable controller message: unknown command '%s'",
945d2e
+                      value);
945d2e
+            status = CRM_EX_PROTOCOL;
945d2e
+            reply = NULL;
945d2e
+        }
945d2e
+    }
945d2e
+
945d2e
+    pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);
945d2e
+}
945d2e
+
945d2e
+pcmk__ipc_methods_t *
945d2e
+pcmk__controld_api_methods()
945d2e
+{
945d2e
+    pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));
945d2e
+
945d2e
+    if (cmds != NULL) {
945d2e
+        cmds->new_data = new_data;
945d2e
+        cmds->free_data = free_data;
945d2e
+        cmds->post_connect = post_connect;
945d2e
+        cmds->reply_expected = reply_expected;
945d2e
+        cmds->dispatch = dispatch;
945d2e
+    }
945d2e
+    return cmds;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \internal
945d2e
+ * \brief Create XML for a controller IPC request
945d2e
+ *
945d2e
+ * \param[in] api       Controller connection
945d2e
+ * \param[in] op        Controller IPC command name
945d2e
+ * \param[in] node      Node name to set as destination host
945d2e
+ * \param[in] msg_data  XML to attach to request as message data
945d2e
+ *
945d2e
+ * \return Newly allocated XML for request
945d2e
+ */
945d2e
+static xmlNode *
945d2e
+create_controller_request(pcmk_ipc_api_t *api, const char *op,
945d2e
+                          const char *node, xmlNode *msg_data)
945d2e
+{
945d2e
+    struct controld_api_private_s *private = api->api_data;
945d2e
+    const char *sys_to = NULL;
945d2e
+
945d2e
+    if ((node == NULL) && !strcmp(op, CRM_OP_PING)) {
945d2e
+        sys_to = CRM_SYSTEM_DC;
945d2e
+    } else {
945d2e
+        sys_to = CRM_SYSTEM_CRMD;
945d2e
+    }
945d2e
+    return create_request(op, msg_data, node, sys_to,
945d2e
+                          (crm_system_name? crm_system_name : "client"),
945d2e
+                          private->client_uuid);
945d2e
+}
945d2e
+
945d2e
+// \return Standard Pacemaker return code
945d2e
+static int
945d2e
+send_controller_request(pcmk_ipc_api_t *api, xmlNode *request,
945d2e
+                        bool reply_is_expected)
945d2e
+{
945d2e
+    int rc;
945d2e
+
945d2e
+    if (crm_element_value(request, XML_ATTR_REFERENCE) == NULL) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+    rc = pcmk__send_ipc_request(api, request);
945d2e
+    if ((rc == pcmk_rc_ok) && reply_is_expected) {
945d2e
+        struct controld_api_private_s *private = api->api_data;
945d2e
+
945d2e
+        private->replies_expected++;
945d2e
+    }
945d2e
+    return rc;
945d2e
+}
945d2e
+
945d2e
+static xmlNode *
945d2e
+create_reprobe_message_data(const char *target_node, const char *router_node)
945d2e
+{
945d2e
+    xmlNode *msg_data;
945d2e
+
945d2e
+    msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE);
945d2e
+    crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
945d2e
+    if ((router_node != NULL) && safe_str_neq(router_node, target_node)) {
945d2e
+        crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
945d2e
+    }
945d2e
+    return msg_data;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Send a reprobe controller operation
945d2e
+ *
945d2e
+ * \param[in] api          Controller connection
945d2e
+ * \param[in] target_node  Name of node to reprobe
945d2e
+ * \param[in] router_node  Router node for host
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ * \note Event callback will get a reply of type pcmk_controld_reply_reprobe.
945d2e
+ */
945d2e
+int
945d2e
+pcmk_controld_api_reprobe(pcmk_ipc_api_t *api, const char *target_node,
945d2e
+                          const char *router_node)
945d2e
+{
945d2e
+    xmlNode *request;
945d2e
+    xmlNode *msg_data;
945d2e
+    int rc = pcmk_rc_ok;
945d2e
+
945d2e
+    if (api == NULL) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+    if (router_node == NULL) {
945d2e
+        router_node = target_node;
945d2e
+    }
945d2e
+    crm_debug("Sending %s IPC request to reprobe %s via %s",
945d2e
+              pcmk_ipc_name(api, true), crm_str(target_node),
945d2e
+              crm_str(router_node));
945d2e
+    msg_data = create_reprobe_message_data(target_node, router_node);
945d2e
+    request = create_controller_request(api, CRM_OP_REPROBE, router_node,
945d2e
+                                        msg_data);
945d2e
+    rc = send_controller_request(api, request, true);
945d2e
+    free_xml(msg_data);
945d2e
+    free_xml(request);
945d2e
+    return rc;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Send a "node info" controller operation
945d2e
+ *
945d2e
+ * \param[in] api          Controller connection
945d2e
+ * \param[in] nodeid       ID of node to get info for (or 0 for local node)
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ * \note Event callback will get a reply of type pcmk_controld_reply_info.
945d2e
+ */
945d2e
+int
945d2e
+pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid)
945d2e
+{
945d2e
+    xmlNode *request;
945d2e
+    int rc = pcmk_rc_ok;
945d2e
+
945d2e
+    request = create_controller_request(api, CRM_OP_NODE_INFO, NULL, NULL);
945d2e
+    if (request == NULL) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+    if (nodeid > 0) {
945d2e
+        crm_xml_set_id(request, "%lu", (unsigned long) nodeid);
945d2e
+    }
945d2e
+
945d2e
+    rc = send_controller_request(api, request, true);
945d2e
+    free_xml(request);
945d2e
+    return rc;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Ask the controller for status
945d2e
+ *
945d2e
+ * \param[in] api          Controller connection
945d2e
+ * \param[in] node_name    Name of node whose status is desired (or NULL for DC)
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ * \note Event callback will get a reply of type pcmk_controld_reply_ping.
945d2e
+ */
945d2e
+int
945d2e
+pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name)
945d2e
+{
945d2e
+    xmlNode *request;
945d2e
+    int rc = pcmk_rc_ok;
945d2e
+
945d2e
+    request = create_controller_request(api, CRM_OP_PING, node_name, NULL);
945d2e
+    if (request == NULL) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+    rc = send_controller_request(api, request, true);
945d2e
+    free_xml(request);
945d2e
+    return rc;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \internal
945d2e
+ * \brief Ask the controller to shut down
945d2e
+ *
945d2e
+ * \param[in] api          Controller connection
945d2e
+ * \param[in] node_name    Name of node whose controller should shut down
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ *
945d2e
+ * \note This capability currently does not work, so the function is considered
945d2e
+ *       internal. It will likely be removed.
945d2e
+ * \note Event callback will not get a reply.
945d2e
+ */
945d2e
+int
945d2e
+pcmk_controld_api_shutdown(pcmk_ipc_api_t *api, const char *node_name)
945d2e
+{
945d2e
+    xmlNode *request;
945d2e
+    int rc = pcmk_rc_ok;
945d2e
+
945d2e
+    request = create_controller_request(api, CRM_OP_SHUTDOWN, NULL, NULL);
945d2e
+    if (request == NULL) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+    rc = send_controller_request(api, request, false);
945d2e
+    free_xml(request);
945d2e
+    return rc;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \internal
945d2e
+ * \brief Ask the controller to start a DC election
945d2e
+ *
945d2e
+ * \param[in] api          Controller connection
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ *
945d2e
+ * \note This capability currently does not work, so the function is considered
945d2e
+ *       internal. It will likely be removed.
945d2e
+ * \note Event callback will not get a reply.
945d2e
+ */
945d2e
+int
945d2e
+pcmk_controld_api_start_election(pcmk_ipc_api_t *api)
945d2e
+{
945d2e
+    xmlNode *request;
945d2e
+    int rc = pcmk_rc_ok;
945d2e
+
945d2e
+    request = create_controller_request(api, CRM_OP_VOTE, NULL, NULL);
945d2e
+    if (request == NULL) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+    rc = send_controller_request(api, request, false);
945d2e
+    free_xml(request);
945d2e
+    return rc;
945d2e
+}
945d2e
+
945d2e
+// \return Standard Pacemaker return code
945d2e
+static int
945d2e
+controller_resource_op(pcmk_ipc_api_t *api, const char *op,
945d2e
+                       const char *target_node, const char *router_node,
945d2e
+                       bool cib_only, const char *rsc_id,
945d2e
+                       const char *rsc_long_id, const char *standard,
945d2e
+                       const char *provider, const char *type)
945d2e
+{
945d2e
+    int rc = pcmk_rc_ok;
945d2e
+    char *key;
945d2e
+    xmlNode *request, *msg_data, *xml_rsc, *params;
945d2e
+
945d2e
+    if (api == NULL) {
945d2e
+        return EINVAL;
945d2e
+    }
945d2e
+    if (router_node == NULL) {
945d2e
+        router_node = target_node;
945d2e
+    }
945d2e
+
945d2e
+    msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
945d2e
+
945d2e
+    /* The controller logs the transition key from resource op requests, so we
945d2e
+     * need to have *something* for it.
945d2e
+     * @TODO don't use "crm-resource"
945d2e
+     */
945d2e
+    key = pcmk__transition_key(0, getpid(), 0,
945d2e
+                               "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
945d2e
+    crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
945d2e
+    free(key);
945d2e
+
945d2e
+    crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
945d2e
+    if (safe_str_neq(router_node, target_node)) {
945d2e
+        crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
945d2e
+    }
945d2e
+
945d2e
+    if (cib_only) {
945d2e
+        // Indicate that only the CIB needs to be cleaned
945d2e
+        crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB);
945d2e
+    }
945d2e
+
945d2e
+    xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
945d2e
+    crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
945d2e
+    crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc_long_id);
945d2e
+    crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, standard);
945d2e
+    crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, provider);
945d2e
+    crm_xml_add(xml_rsc, XML_ATTR_TYPE, type);
945d2e
+
945d2e
+    params = create_xml_node(msg_data, XML_TAG_ATTRS);
945d2e
+    crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
945d2e
+
945d2e
+    // The controller parses the timeout from the request
945d2e
+    key = crm_meta_name(XML_ATTR_TIMEOUT);
945d2e
+    crm_xml_add(params, key, "60000");  /* 1 minute */ //@TODO pass as arg
945d2e
+    free(key);
945d2e
+
945d2e
+    request = create_controller_request(api, op, router_node, msg_data);
945d2e
+    rc = send_controller_request(api, request, true);
945d2e
+    free_xml(msg_data);
945d2e
+    free_xml(request);
945d2e
+    return rc;
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Ask the controller to fail a resource
945d2e
+ *
945d2e
+ * \param[in] api          Controller connection
945d2e
+ * \param[in] target_node  Name of node resource is on
945d2e
+ * \param[in] router_node  Router node for target
945d2e
+ * \param[in] rsc_id       ID of resource to fail
945d2e
+ * \param[in] rsc_long_id  Long ID of resource (if any)
945d2e
+ * \param[in] standard     Standard of resource
945d2e
+ * \param[in] provider     Provider of resource (if any)
945d2e
+ * \param[in] type         Type of resource to fail
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ * \note Event callback will get a reply of type pcmk_controld_reply_resource.
945d2e
+ */
945d2e
+int
945d2e
+pcmk_controld_api_fail(pcmk_ipc_api_t *api,
945d2e
+                       const char *target_node, const char *router_node,
945d2e
+                       const char *rsc_id, const char *rsc_long_id,
945d2e
+                       const char *standard, const char *provider,
945d2e
+                       const char *type)
945d2e
+{
945d2e
+    crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s",
945d2e
+              pcmk_ipc_name(api, true), crm_str(rsc_id), crm_str(rsc_long_id),
945d2e
+              crm_str(target_node), crm_str(router_node));
945d2e
+    return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node,
945d2e
+                                  router_node, false, rsc_id, rsc_long_id,
945d2e
+                                  standard, provider, type);
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Ask the controller to refresh a resource
945d2e
+ *
945d2e
+ * \param[in] api          Controller connection
945d2e
+ * \param[in] target_node  Name of node resource is on
945d2e
+ * \param[in] router_node  Router node for target
945d2e
+ * \param[in] rsc_id       ID of resource to refresh
945d2e
+ * \param[in] rsc_long_id  Long ID of resource (if any)
945d2e
+ * \param[in] standard     Standard of resource
945d2e
+ * \param[in] provider     Provider of resource (if any)
945d2e
+ * \param[in] type         Type of resource
945d2e
+ * \param[in] cib_only     If true, clean resource from CIB only
945d2e
+ *
945d2e
+ * \return Standard Pacemaker return code
945d2e
+ * \note Event callback will get a reply of type pcmk_controld_reply_resource.
945d2e
+ */
945d2e
+int
945d2e
+pcmk_controld_api_refresh(pcmk_ipc_api_t *api, const char *target_node,
945d2e
+                          const char *router_node,
945d2e
+                          const char *rsc_id, const char *rsc_long_id,
945d2e
+                          const char *standard, const char *provider,
945d2e
+                          const char *type, bool cib_only)
945d2e
+{
945d2e
+    crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s",
945d2e
+              pcmk_ipc_name(api, true), crm_str(rsc_id), crm_str(rsc_long_id),
945d2e
+              crm_str(target_node), crm_str(router_node));
945d2e
+    return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node,
945d2e
+                                  router_node, cib_only, rsc_id, rsc_long_id,
945d2e
+                                  standard, provider, type);
945d2e
+}
945d2e
+
945d2e
+/*!
945d2e
+ * \brief Get the number of IPC replies currently expected from the controller
945d2e
+ *
945d2e
+ * \param[in] api  Controller IPC API connection
945d2e
+ *
945d2e
+ * \return Number of replies expected
945d2e
+ */
945d2e
+unsigned int
945d2e
+pcmk_controld_api_replies_expected(pcmk_ipc_api_t *api)
945d2e
+{
945d2e
+    struct controld_api_private_s *private = api->api_data;
945d2e
+
945d2e
+    return private->replies_expected;
945d2e
+}
945d2e
+
945d2e
+xmlNode *
945d2e
+create_hello_message(const char *uuid, const char *client_name,
945d2e
+                     const char *major_version, const char *minor_version)
945d2e
+{
945d2e
+    xmlNode *hello_node = NULL;
945d2e
+    xmlNode *hello = NULL;
945d2e
+
945d2e
+    if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name)
945d2e
+        || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) {
945d2e
+        crm_err("Could not create IPC hello message from %s (UUID %s): "
945d2e
+                "missing information",
945d2e
+                client_name? client_name : "unknown client",
945d2e
+                uuid? uuid : "unknown");
945d2e
+        return NULL;
945d2e
+    }
945d2e
+
945d2e
+    hello_node = create_xml_node(NULL, XML_TAG_OPTIONS);
945d2e
+    if (hello_node == NULL) {
945d2e
+        crm_err("Could not create IPC hello message from %s (UUID %s): "
945d2e
+                "Message data creation failed", client_name, uuid);
945d2e
+        return NULL;
945d2e
+    }
945d2e
+
945d2e
+    crm_xml_add(hello_node, "major_version", major_version);
945d2e
+    crm_xml_add(hello_node, "minor_version", minor_version);
945d2e
+    crm_xml_add(hello_node, "client_name", client_name);
945d2e
+    crm_xml_add(hello_node, "client_uuid", uuid);
945d2e
+
945d2e
+    hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
945d2e
+    if (hello == NULL) {
945d2e
+        crm_err("Could not create IPC hello message from %s (UUID %s): "
945d2e
+                "Request creation failed", client_name, uuid);
945d2e
+        return NULL;
945d2e
+    }
945d2e
+    free_xml(hello_node);
945d2e
+
945d2e
+    crm_trace("Created hello message from %s (UUID %s)", client_name, uuid);
945d2e
+    return hello;
945d2e
+}
945d2e
-- 
945d2e
1.8.3.1
945d2e
945d2e
945d2e
From 1d1b34664b64f6805aeaf4d8e29e77aa8f59b4fc Mon Sep 17 00:00:00 2001
945d2e
From: Ken Gaillot <kgaillot@redhat.com>
945d2e
Date: Tue, 7 Apr 2020 15:43:01 -0500
945d2e
Subject: [PATCH 3/6] Refactor: tools: convert crm_resource to use new
945d2e
 controller IPC model
945d2e
945d2e
---
945d2e
 tools/Makefile.am               |   3 +-
945d2e
 tools/crm_resource.c            | 138 +++++++------
945d2e
 tools/crm_resource.h            |   9 +-
945d2e
 tools/crm_resource_controller.c | 425 ----------------------------------------
945d2e
 tools/crm_resource_controller.h | 198 -------------------
945d2e
 tools/crm_resource_runtime.c    |  30 +--
945d2e
 6 files changed, 96 insertions(+), 707 deletions(-)
945d2e
 delete mode 100644 tools/crm_resource_controller.c
945d2e
 delete mode 100644 tools/crm_resource_controller.h
945d2e
945d2e
diff --git a/tools/Makefile.am b/tools/Makefile.am
945d2e
index c822a8c..4609b0f 100644
945d2e
--- a/tools/Makefile.am
945d2e
+++ b/tools/Makefile.am
945d2e
@@ -12,7 +12,7 @@ if BUILD_SYSTEMD
945d2e
 systemdsystemunit_DATA	= crm_mon.service
945d2e
 endif
945d2e
 
945d2e
-noinst_HEADERS		= crm_mon.h crm_resource.h crm_resource_controller.h
945d2e
+noinst_HEADERS		= crm_mon.h crm_resource.h
945d2e
 
945d2e
 pcmkdir			= $(datadir)/$(PACKAGE)
945d2e
 pcmk_DATA		= report.common report.collector
945d2e
@@ -115,7 +115,6 @@ crm_attribute_LDADD	= $(top_builddir)/lib/cluster/libcrmcluster.la	\
945d2e
 
945d2e
 crm_resource_SOURCES	= crm_resource.c		\
945d2e
 			  crm_resource_ban.c		\
945d2e
-			  crm_resource_controller.c	\
945d2e
 			  crm_resource_print.c		\
945d2e
 			  crm_resource_runtime.c
945d2e
 crm_resource_LDADD	= $(top_builddir)/lib/pengine/libpe_rules.la  	\
945d2e
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
945d2e
index 6853ad5..c8c1cfa 100644
945d2e
--- a/tools/crm_resource.c
945d2e
+++ b/tools/crm_resource.c
945d2e
@@ -11,62 +11,70 @@
945d2e
 #include <pacemaker-internal.h>
945d2e
 
945d2e
 #include <sys/param.h>
945d2e
-
945d2e
-#include <crm/crm.h>
945d2e
-#include <crm/stonith-ng.h>
945d2e
-
945d2e
 #include <stdio.h>
945d2e
 #include <sys/types.h>
945d2e
 #include <unistd.h>
945d2e
-
945d2e
 #include <stdlib.h>
945d2e
 #include <errno.h>
945d2e
 #include <fcntl.h>
945d2e
 #include <libgen.h>
945d2e
 #include <time.h>
945d2e
 
945d2e
+#include <crm/crm.h>
945d2e
+#include <crm/stonith-ng.h>
945d2e
+#include <crm/common/ipc_controld.h>
945d2e
+
945d2e
 bool BE_QUIET = FALSE;
945d2e
 bool scope_master = FALSE;
945d2e
 int cib_options = cib_sync_call;
945d2e
-
945d2e
-static GMainLoop *mainloop = NULL;
945d2e
+static crm_exit_t exit_code = CRM_EX_OK;
945d2e
 
945d2e
 // Things that should be cleaned up on exit
945d2e
+static GMainLoop *mainloop = NULL;
945d2e
 static cib_t *cib_conn = NULL;
945d2e
-static pcmk_controld_api_t *controld_api = NULL;
945d2e
+static pcmk_ipc_api_t *controld_api = NULL;
945d2e
 static pe_working_set_t *data_set = NULL;
945d2e
 
945d2e
 #define MESSAGE_TIMEOUT_S 60
945d2e
 
945d2e
 // Clean up and exit
945d2e
 static crm_exit_t
945d2e
-bye(crm_exit_t exit_code)
945d2e
+bye(crm_exit_t ec)
945d2e
 {
945d2e
-    static bool crm_resource_shutdown_flag = false;
945d2e
-
945d2e
-    if (crm_resource_shutdown_flag) {
945d2e
-        // Allow usage like "return bye(...);"
945d2e
-        return exit_code;
945d2e
-    }
945d2e
-    crm_resource_shutdown_flag = true;
945d2e
-
945d2e
     if (cib_conn != NULL) {
945d2e
         cib_t *save_cib_conn = cib_conn;
945d2e
 
945d2e
-        cib_conn = NULL;
945d2e
+        cib_conn = NULL; // Ensure we can't free this twice
945d2e
         save_cib_conn->cmds->signoff(save_cib_conn);
945d2e
         cib_delete(save_cib_conn);
945d2e
     }
945d2e
     if (controld_api != NULL) {
945d2e
-        pcmk_controld_api_t *save_controld_api = controld_api;
945d2e
+        pcmk_ipc_api_t *save_controld_api = controld_api;
945d2e
 
945d2e
-        controld_api = NULL;
945d2e
-        pcmk_free_controld_api(save_controld_api);
945d2e
+        controld_api = NULL; // Ensure we can't free this twice
945d2e
+        pcmk_free_ipc_api(save_controld_api);
945d2e
+    }
945d2e
+    if (mainloop != NULL) {
945d2e
+        g_main_loop_unref(mainloop);
945d2e
+        mainloop = NULL;
945d2e
     }
945d2e
     pe_free_working_set(data_set);
945d2e
     data_set = NULL;
945d2e
-    crm_exit(exit_code);
945d2e
-    return exit_code;
945d2e
+    crm_exit(ec);
945d2e
+    return ec;
945d2e
+}
945d2e
+
945d2e
+static void
945d2e
+quit_main_loop(crm_exit_t ec)
945d2e
+{
945d2e
+    exit_code = ec;
945d2e
+    if (mainloop != NULL) {
945d2e
+        GMainLoop *mloop = mainloop;
945d2e
+
945d2e
+        mainloop = NULL; // Don't re-enter this block
945d2e
+        pcmk_quit_main_loop(mloop, 10);
945d2e
+        g_main_loop_unref(mloop);
945d2e
+    }
945d2e
 }
945d2e
 
945d2e
 static gboolean
945d2e
@@ -76,39 +84,54 @@ resource_ipc_timeout(gpointer data)
945d2e
     fprintf(stderr, "\nAborting because no messages received in %d seconds\n",
945d2e
             MESSAGE_TIMEOUT_S);
945d2e
     crm_err("No messages received in %d seconds", MESSAGE_TIMEOUT_S);
945d2e
-    bye(CRM_EX_TIMEOUT);
945d2e
+    quit_main_loop(CRM_EX_TIMEOUT);
945d2e
     return FALSE;
945d2e
 }
945d2e
 
945d2e
 static void
945d2e
-handle_controller_reply(pcmk_controld_api_t *capi, void *api_data,
945d2e
-                        void *user_data)
945d2e
+controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
945d2e
+                          crm_exit_t status, void *event_data, void *user_data)
945d2e
 {
945d2e
-    fprintf(stderr, ".");
945d2e
-    if ((capi->replies_expected(capi) == 0)
945d2e
-        && mainloop && g_main_loop_is_running(mainloop)) {
945d2e
-        fprintf(stderr, " OK\n");
945d2e
-        crm_debug("Got all the replies we expected");
945d2e
-        bye(CRM_EX_OK);
945d2e
-    }
945d2e
-}
945d2e
+    switch (event_type) {
945d2e
+        case pcmk_ipc_event_disconnect:
945d2e
+            if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
945d2e
+                crm_info("Connection to controller was terminated");
945d2e
+            }
945d2e
+            quit_main_loop(exit_code);
945d2e
+            break;
945d2e
 
945d2e
-static void
945d2e
-handle_controller_drop(pcmk_controld_api_t *capi, void *api_data,
945d2e
-                       void *user_data)
945d2e
-{
945d2e
-    crm_info("Connection to controller was terminated");
945d2e
-    bye(CRM_EX_DISCONNECT);
945d2e
+        case pcmk_ipc_event_reply:
945d2e
+            if (status != CRM_EX_OK) {
945d2e
+                fprintf(stderr, "\nError: bad reply from controller: %s\n",
945d2e
+                        crm_exit_str(status));
945d2e
+                pcmk_disconnect_ipc(api);
945d2e
+                quit_main_loop(status);
945d2e
+            } else {
945d2e
+                fprintf(stderr, ".");
945d2e
+                if ((pcmk_controld_api_replies_expected(api) == 0)
945d2e
+                    && mainloop && g_main_loop_is_running(mainloop)) {
945d2e
+                    fprintf(stderr, " OK\n");
945d2e
+                    crm_debug("Got all the replies we expected");
945d2e
+                    pcmk_disconnect_ipc(api);
945d2e
+                    quit_main_loop(CRM_EX_OK);
945d2e
+                }
945d2e
+            }
945d2e
+            break;
945d2e
+
945d2e
+        default:
945d2e
+            break;
945d2e
+    }
945d2e
 }
945d2e
 
945d2e
 static void
945d2e
-start_mainloop(pcmk_controld_api_t *capi)
945d2e
+start_mainloop(pcmk_ipc_api_t *capi)
945d2e
 {
945d2e
-    if (capi->replies_expected(capi) > 0) {
945d2e
-        unsigned int count = capi->replies_expected(capi);
945d2e
+    unsigned int count = pcmk_controld_api_replies_expected(capi);
945d2e
 
945d2e
+    if (count > 0) {
945d2e
         fprintf(stderr, "Waiting for %d %s from the controller",
945d2e
                 count, pcmk__plural_alt(count, "reply", "replies"));
945d2e
+        exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
945d2e
         mainloop = g_main_loop_new(NULL, FALSE);
945d2e
         g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
945d2e
         g_main_loop_run(mainloop);
945d2e
@@ -664,7 +687,6 @@ main(int argc, char **argv)
945d2e
     int argerr = 0;
945d2e
     int flag;
945d2e
     int find_flags = 0;           // Flags to use when searching for resource
945d2e
-    crm_exit_t exit_code = CRM_EX_OK;
945d2e
 
945d2e
     crm_log_cli_init("crm_resource");
945d2e
     pcmk__set_cli_options(NULL, "<query>|<command> [options]", long_options,
945d2e
@@ -1151,21 +1173,15 @@ main(int argc, char **argv)
945d2e
 
945d2e
     // Establish a connection to the controller if needed
945d2e
     if (require_crmd) {
945d2e
-        char *client_uuid;
945d2e
-        pcmk_controld_api_cb_t dispatch_cb = {
945d2e
-            handle_controller_reply, NULL
945d2e
-        };
945d2e
-        pcmk_controld_api_cb_t destroy_cb = {
945d2e
-            handle_controller_drop, NULL
945d2e
-        };
945d2e
-
945d2e
-
945d2e
-        client_uuid = pcmk__getpid_s();
945d2e
-        controld_api = pcmk_new_controld_api(crm_system_name, client_uuid);
945d2e
-        free(client_uuid);
945d2e
-
945d2e
-        rc = controld_api->connect(controld_api, true, &dispatch_cb,
945d2e
-                                   &destroy_cb);
945d2e
+        rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
945d2e
+        if (rc != pcmk_rc_ok) {
945d2e
+            CMD_ERR("Error connecting to the controller: %s", pcmk_rc_str(rc));
945d2e
+            rc = pcmk_rc2legacy(rc);
945d2e
+            goto bail;
945d2e
+        }
945d2e
+        pcmk_register_ipc_callback(controld_api, controller_event_callback,
945d2e
+                                   NULL);
945d2e
+        rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main);
945d2e
         if (rc != pcmk_rc_ok) {
945d2e
             CMD_ERR("Error connecting to the controller: %s", pcmk_rc_str(rc));
945d2e
             rc = pcmk_rc2legacy(rc);
945d2e
@@ -1525,8 +1541,8 @@ main(int argc, char **argv)
945d2e
                                                           NULL, NULL, NULL,
945d2e
                                                           NULL, attr_options));
945d2e
 
945d2e
-        if (controld_api->reprobe(controld_api, host_uname,
945d2e
-                                  router_node) == pcmk_rc_ok) {
945d2e
+        if (pcmk_controld_api_reprobe(controld_api, host_uname,
945d2e
+                                      router_node) == pcmk_rc_ok) {
945d2e
             start_mainloop(controld_api);
945d2e
         }
945d2e
 
945d2e
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
945d2e
index 0bf7bee..cb7f506 100644
945d2e
--- a/tools/crm_resource.h
945d2e
+++ b/tools/crm_resource.h
945d2e
@@ -21,7 +21,6 @@
945d2e
 #include <crm/pengine/status.h>
945d2e
 #include <crm/pengine/internal.h>
945d2e
 #include <pacemaker-internal.h>
945d2e
-#include "crm_resource_controller.h"
945d2e
 
945d2e
 extern bool print_pending;
945d2e
 
945d2e
@@ -36,8 +35,6 @@ extern char *move_lifetime;
945d2e
 
945d2e
 extern const char *attr_set_type;
945d2e
 
945d2e
-extern pcmk_controld_api_cb_t controld_api_cb;
945d2e
-
945d2e
 /* ban */
945d2e
 int cli_resource_prefer(const char *rsc_id, const char *host, cib_t * cib_conn);
945d2e
 int cli_resource_ban(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn);
945d2e
@@ -63,16 +60,16 @@ int cli_resource_print_operations(const char *rsc_id, const char *host_uname, bo
945d2e
 
945d2e
 /* runtime */
945d2e
 void cli_resource_check(cib_t * cib, pe_resource_t *rsc);
945d2e
-int cli_resource_fail(pcmk_controld_api_t *controld_api,
945d2e
+int cli_resource_fail(pcmk_ipc_api_t *controld_api,
945d2e
                       const char *host_uname, const char *rsc_id,
945d2e
                       pe_working_set_t *data_set);
945d2e
 int cli_resource_search(pe_resource_t *rsc, const char *requested_name,
945d2e
                         pe_working_set_t *data_set);
945d2e
-int cli_resource_delete(pcmk_controld_api_t *controld_api,
945d2e
+int cli_resource_delete(pcmk_ipc_api_t *controld_api,
945d2e
                         const char *host_uname, pe_resource_t *rsc,
945d2e
                         const char *operation, const char *interval_spec,
945d2e
                         bool just_failures, pe_working_set_t *data_set);
945d2e
-int cli_cleanup_all(pcmk_controld_api_t *controld_api, const char *node_name,
945d2e
+int cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
945d2e
                     const char *operation, const char *interval_spec,
945d2e
                     pe_working_set_t *data_set);
945d2e
 int cli_resource_restart(pe_resource_t *rsc, const char *host, int timeout_ms,
945d2e
diff --git a/tools/crm_resource_controller.c b/tools/crm_resource_controller.c
945d2e
deleted file mode 100644
945d2e
index 994a7be..0000000
945d2e
--- a/tools/crm_resource_controller.c
945d2e
+++ /dev/null
945d2e
@@ -1,425 +0,0 @@
945d2e
-/*
945d2e
- * Copyright 2020 the Pacemaker project contributors
945d2e
- *
945d2e
- * The version control history for this file may have further details.
945d2e
- *
945d2e
- * This source code is licensed under the GNU General Public License version 2
945d2e
- * or later (GPLv2+) WITHOUT ANY WARRANTY.
945d2e
- */
945d2e
-
945d2e
-#include <crm_internal.h>
945d2e
-#include <stdio.h>
945d2e
-#include <errno.h>
945d2e
-#include "crm_resource.h"
945d2e
-
945d2e
-// API object's private members
945d2e
-struct controller_private {
945d2e
-    char *client_name;              // Client name to use with IPC
945d2e
-    char *client_uuid;              // Client UUID to use with IPC
945d2e
-    mainloop_io_t *source;          // If main loop used, I/O source for IPC
945d2e
-    crm_ipc_t *ipc;                 // IPC connection to controller
945d2e
-    int replies_expected;           // How many controller replies are expected
945d2e
-    pcmk_controld_api_cb_t dispatch_cb; // Caller's registered dispatch callback
945d2e
-    pcmk_controld_api_cb_t destroy_cb;  // Caller's registered destroy callback
945d2e
-};
945d2e
-
945d2e
-static void
945d2e
-call_client_callback(pcmk_controld_api_t *api, pcmk_controld_api_cb_t *cb,
945d2e
-                     void *api_data)
945d2e
-{
945d2e
-    if ((cb != NULL) && (cb->callback != NULL)) {
945d2e
-        cb->callback(api, api_data, cb->user_data);
945d2e
-    }
945d2e
-}
945d2e
-
945d2e
-/*
945d2e
- * IPC callbacks when used with main loop
945d2e
- */
945d2e
-
945d2e
-static void
945d2e
-controller_ipc_destroy(gpointer user_data)
945d2e
-{
945d2e
-    pcmk_controld_api_t *api = user_data;
945d2e
-    struct controller_private *private = api->private;
945d2e
-
945d2e
-    private->ipc = NULL;
945d2e
-    private->source = NULL;
945d2e
-    call_client_callback(api, &(private->destroy_cb), NULL);
945d2e
-}
945d2e
-
945d2e
-// \return < 0 if connection is no longer required, >= 0 if it is
945d2e
-static int
945d2e
-controller_ipc_dispatch(const char *buffer, ssize_t length, gpointer user_data)
945d2e
-{
945d2e
-    xmlNode *msg = NULL;
945d2e
-    pcmk_controld_api_t *api = user_data;
945d2e
-
945d2e
-    CRM_CHECK(buffer && api && api->private, return 0);
945d2e
-
945d2e
-    msg = string2xml(buffer);
945d2e
-    if (msg == NULL) {
945d2e
-        crm_warn("Received malformed controller IPC message");
945d2e
-    } else {
945d2e
-        struct controller_private *private = api->private;
945d2e
-
945d2e
-        crm_log_xml_trace(msg, "controller-reply");
945d2e
-        private->replies_expected--;
945d2e
-        call_client_callback(api, &(private->dispatch_cb),
945d2e
-                             get_message_xml(msg, F_CRM_DATA));
945d2e
-        free_xml(msg);
945d2e
-    }
945d2e
-    return 0;
945d2e
-}
945d2e
-
945d2e
-/*
945d2e
- * IPC utilities
945d2e
- */
945d2e
-
945d2e
-// \return Standard Pacemaker return code
945d2e
-static int
945d2e
-send_hello(crm_ipc_t *ipc, const char *client_name, const char *client_uuid)
945d2e
-{
945d2e
-    xmlNode *hello = create_hello_message(client_uuid, client_name, "0", "1");
945d2e
-    int rc = crm_ipc_send(ipc, hello, 0, 0, NULL);
945d2e
-
945d2e
-    free_xml(hello);
945d2e
-    if (rc < 0) {
945d2e
-        rc = pcmk_legacy2rc(rc);
945d2e
-        crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s",
945d2e
-                 CRM_SYSTEM_CRMD /* ipc->name */,
945d2e
-                 pcmk_rc_str(rc), rc);
945d2e
-        return rc;
945d2e
-    }
945d2e
-    crm_debug("Sent IPC hello to %s", CRM_SYSTEM_CRMD /* ipc->name */);
945d2e
-    return pcmk_rc_ok;
945d2e
-}
945d2e
-
945d2e
-// \return Standard Pacemaker return code
945d2e
-static int
945d2e
-send_controller_request(pcmk_controld_api_t *api, const char *op,
945d2e
-                        xmlNode *msg_data, const char *node)
945d2e
-{
945d2e
-    int rc;
945d2e
-    struct controller_private *private = api->private;
945d2e
-    xmlNode *cmd = create_request(op, msg_data, node, CRM_SYSTEM_CRMD,
945d2e
-                                  private->client_name, private->client_uuid);
945d2e
-    const char *reference = crm_element_value(cmd, XML_ATTR_REFERENCE);
945d2e
-
945d2e
-    if ((cmd == NULL) || (reference == NULL)) {
945d2e
-        return EINVAL;
945d2e
-    }
945d2e
-
945d2e
-    //@TODO pass as args? 0=crm_ipc_flags, 0=timeout_ms (default 5s), NULL=reply
945d2e
-    crm_log_xml_trace(cmd, "controller-request");
945d2e
-    rc = crm_ipc_send(private->ipc, cmd, 0, 0, NULL);
945d2e
-    free_xml(cmd);
945d2e
-    if (rc < 0) {
945d2e
-        return pcmk_legacy2rc(rc);
945d2e
-    }
945d2e
-    private->replies_expected++;
945d2e
-    return pcmk_rc_ok;
945d2e
-}
945d2e
-
945d2e
-/*
945d2e
- * pcmk_controld_api_t methods
945d2e
- */
945d2e
-
945d2e
-static int
945d2e
-controller_connect_mainloop(pcmk_controld_api_t *api)
945d2e
-{
945d2e
-    struct controller_private *private = api->private;
945d2e
-    struct ipc_client_callbacks callbacks = {
945d2e
-        .dispatch = controller_ipc_dispatch,
945d2e
-        .destroy = controller_ipc_destroy,
945d2e
-    };
945d2e
-
945d2e
-    private->source = mainloop_add_ipc_client(CRM_SYSTEM_CRMD,
945d2e
-                                              G_PRIORITY_DEFAULT, 0, api,
945d2e
-                                              &callbacks);
945d2e
-    if (private->source == NULL) {
945d2e
-        return ENOTCONN;
945d2e
-    }
945d2e
-
945d2e
-    private->ipc = mainloop_get_ipc_client(private->source);
945d2e
-    if (private->ipc == NULL) {
945d2e
-        (void) api->disconnect(api);
945d2e
-        return ENOTCONN;
945d2e
-    }
945d2e
-
945d2e
-    crm_debug("Connected to %s IPC (attaching to main loop)", CRM_SYSTEM_CRMD);
945d2e
-    return pcmk_rc_ok;
945d2e
-}
945d2e
-
945d2e
-static int
945d2e
-controller_connect_no_mainloop(pcmk_controld_api_t *api)
945d2e
-{
945d2e
-    struct controller_private *private = api->private;
945d2e
-
945d2e
-    private->ipc = crm_ipc_new(CRM_SYSTEM_CRMD, 0);
945d2e
-    if (private->ipc == NULL) {
945d2e
-        return ENOTCONN;
945d2e
-    }
945d2e
-    if (!crm_ipc_connect(private->ipc)) {
945d2e
-        crm_ipc_close(private->ipc);
945d2e
-        crm_ipc_destroy(private->ipc);
945d2e
-        private->ipc = NULL;
945d2e
-        return errno;
945d2e
-    }
945d2e
-    /* @TODO caller needs crm_ipc_get_fd(private->ipc); either add method for
945d2e
-     * that, or replace use_mainloop with int *fd
945d2e
-     */
945d2e
-    crm_debug("Connected to %s IPC", CRM_SYSTEM_CRMD);
945d2e
-    return pcmk_rc_ok;
945d2e
-}
945d2e
-
945d2e
-static void
945d2e
-set_callback(pcmk_controld_api_cb_t *dest, pcmk_controld_api_cb_t *source)
945d2e
-{
945d2e
-    if (source) {
945d2e
-        dest->callback  = source->callback;
945d2e
-        dest->user_data = source->user_data;
945d2e
-    }
945d2e
-}
945d2e
-
945d2e
-static int
945d2e
-controller_api_connect(pcmk_controld_api_t *api, bool use_mainloop,
945d2e
-                       pcmk_controld_api_cb_t *dispatch_cb,
945d2e
-                       pcmk_controld_api_cb_t *destroy_cb)
945d2e
-{
945d2e
-    int rc = pcmk_rc_ok;
945d2e
-    struct controller_private *private;
945d2e
-
945d2e
-    if (api == NULL) {
945d2e
-        return EINVAL;
945d2e
-    }
945d2e
-    private = api->private;
945d2e
-
945d2e
-    set_callback(&(private->dispatch_cb), dispatch_cb);
945d2e
-    set_callback(&(private->destroy_cb), destroy_cb);
945d2e
-
945d2e
-    if (private->ipc != NULL) {
945d2e
-        return pcmk_rc_ok; // already connected
945d2e
-    }
945d2e
-
945d2e
-    if (use_mainloop) {
945d2e
-        rc = controller_connect_mainloop(api);
945d2e
-    } else {
945d2e
-        rc = controller_connect_no_mainloop(api);
945d2e
-    }
945d2e
-    if (rc != pcmk_rc_ok) {
945d2e
-        return rc;
945d2e
-    }
945d2e
-
945d2e
-    rc = send_hello(private->ipc, private->client_name, private->client_uuid);
945d2e
-    if (rc != pcmk_rc_ok) {
945d2e
-        (void) api->disconnect(api);
945d2e
-    }
945d2e
-    return rc;
945d2e
-}
945d2e
-
945d2e
-static int
945d2e
-controller_api_disconnect(pcmk_controld_api_t *api)
945d2e
-{
945d2e
-    struct controller_private *private = api->private;
945d2e
-
945d2e
-    if (private->source != NULL) {
945d2e
-        // Attached to main loop
945d2e
-        mainloop_del_ipc_client(private->source);
945d2e
-        private->source = NULL;
945d2e
-        private->ipc = NULL;
945d2e
-
945d2e
-    } else if (private->ipc != NULL) {
945d2e
-        // Not attached to main loop
945d2e
-        crm_ipc_t *ipc = private->ipc;
945d2e
-
945d2e
-        private->ipc = NULL;
945d2e
-        crm_ipc_close(ipc);
945d2e
-        crm_ipc_destroy(ipc);
945d2e
-    }
945d2e
-    crm_debug("Disconnected from %s IPC", CRM_SYSTEM_CRMD /* ipc->name */);
945d2e
-    return pcmk_rc_ok;
945d2e
-}
945d2e
-
945d2e
-//@TODO dispatch function for non-mainloop a la stonith_dispatch()
945d2e
-//@TODO convenience retry-connect function a la stonith_api_connect_retry()
945d2e
-
945d2e
-static unsigned int
945d2e
-controller_api_replies_expected(pcmk_controld_api_t *api)
945d2e
-{
945d2e
-    if (api != NULL) {
945d2e
-        struct controller_private *private = api->private;
945d2e
-
945d2e
-        return private->replies_expected;
945d2e
-    }
945d2e
-    return 0;
945d2e
-}
945d2e
-
945d2e
-static xmlNode *
945d2e
-create_reprobe_message_data(const char *target_node, const char *router_node)
945d2e
-{
945d2e
-    xmlNode *msg_data;
945d2e
-
945d2e
-    msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE);
945d2e
-    crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
945d2e
-    if ((router_node != NULL) && safe_str_neq(router_node, target_node)) {
945d2e
-        crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
945d2e
-    }
945d2e
-    return msg_data;
945d2e
-}
945d2e
-
945d2e
-static int
945d2e
-controller_api_reprobe(pcmk_controld_api_t *api, const char *target_node,
945d2e
-                       const char *router_node)
945d2e
-{
945d2e
-    int rc = EINVAL;
945d2e
-
945d2e
-    if (api != NULL) {
945d2e
-        xmlNode *msg_data;
945d2e
-
945d2e
-        crm_debug("Sending %s IPC request to reprobe %s via %s",
945d2e
-                  CRM_SYSTEM_CRMD, crm_str(target_node), crm_str(router_node));
945d2e
-        msg_data = create_reprobe_message_data(target_node, router_node);
945d2e
-        rc = send_controller_request(api, CRM_OP_REPROBE, msg_data,
945d2e
-                                     (router_node? router_node : target_node));
945d2e
-        free_xml(msg_data);
945d2e
-    }
945d2e
-    return rc;
945d2e
-}
945d2e
-
945d2e
-// \return Standard Pacemaker return code
945d2e
-static int
945d2e
-controller_resource_op(pcmk_controld_api_t *api, const char *op,
945d2e
-                       const char *target_node, const char *router_node,
945d2e
-                       bool cib_only, const char *rsc_id,
945d2e
-                       const char *rsc_long_id, const char *standard,
945d2e
-                       const char *provider, const char *type)
945d2e
-{
945d2e
-    int rc;
945d2e
-    char *key;
945d2e
-    xmlNode *msg_data, *xml_rsc, *params;
945d2e
-
945d2e
-    if (api == NULL) {
945d2e
-        return EINVAL;
945d2e
-    }
945d2e
-    if (router_node == NULL) {
945d2e
-        router_node = target_node;
945d2e
-    }
945d2e
-
945d2e
-    msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
945d2e
-
945d2e
-    /* The controller logs the transition key from resource op requests, so we
945d2e
-     * need to have *something* for it.
945d2e
-     */
945d2e
-    key = pcmk__transition_key(0, getpid(), 0,
945d2e
-                               "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
945d2e
-    crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
945d2e
-    free(key);
945d2e
-
945d2e
-    crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
945d2e
-    if (safe_str_neq(router_node, target_node)) {
945d2e
-        crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
945d2e
-    }
945d2e
-
945d2e
-    if (cib_only) {
945d2e
-        // Indicate that only the CIB needs to be cleaned
945d2e
-        crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB);
945d2e
-    }
945d2e
-
945d2e
-    xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
945d2e
-    crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
945d2e
-    crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc_long_id);
945d2e
-    crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, standard);
945d2e
-    crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, provider);
945d2e
-    crm_xml_add(xml_rsc, XML_ATTR_TYPE, type);
945d2e
-
945d2e
-    params = create_xml_node(msg_data, XML_TAG_ATTRS);
945d2e
-    crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
945d2e
-
945d2e
-    // The controller parses the timeout from the request
945d2e
-    key = crm_meta_name(XML_ATTR_TIMEOUT);
945d2e
-    crm_xml_add(params, key, "60000");  /* 1 minute */ //@TODO pass as arg
945d2e
-    free(key);
945d2e
-
945d2e
-    rc = send_controller_request(api, op, msg_data, router_node);
945d2e
-    free_xml(msg_data);
945d2e
-    return rc;
945d2e
-}
945d2e
-
945d2e
-static int
945d2e
-controller_api_fail_resource(pcmk_controld_api_t *api,
945d2e
-                             const char *target_node, const char *router_node,
945d2e
-                             const char *rsc_id, const char *rsc_long_id,
945d2e
-                             const char *standard, const char *provider,
945d2e
-                             const char *type)
945d2e
-{
945d2e
-    crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s",
945d2e
-              CRM_SYSTEM_CRMD, crm_str(rsc_id), crm_str(rsc_long_id),
945d2e
-              crm_str(target_node), crm_str(router_node));
945d2e
-    return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node,
945d2e
-                                  router_node, false, rsc_id, rsc_long_id,
945d2e
-                                  standard, provider, type);
945d2e
-}
945d2e
-
945d2e
-static int
945d2e
-controller_api_refresh_resource(pcmk_controld_api_t *api,
945d2e
-                                const char *target_node,
945d2e
-                                const char *router_node,
945d2e
-                                const char *rsc_id, const char *rsc_long_id,
945d2e
-                                const char *standard, const char *provider,
945d2e
-                                const char *type, bool cib_only)
945d2e
-{
945d2e
-    crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s",
945d2e
-              CRM_SYSTEM_CRMD, crm_str(rsc_id), crm_str(rsc_long_id),
945d2e
-              crm_str(target_node), crm_str(router_node));
945d2e
-    return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node,
945d2e
-                                  router_node, cib_only, rsc_id, rsc_long_id,
945d2e
-                                  standard, provider, type);
945d2e
-}
945d2e
-
945d2e
-pcmk_controld_api_t *
945d2e
-pcmk_new_controld_api(const char *client_name, const char *client_uuid)
945d2e
-{
945d2e
-    struct controller_private *private;
945d2e
-    pcmk_controld_api_t *api = calloc(1, sizeof(pcmk_controld_api_t));
945d2e
-
945d2e
-    CRM_ASSERT(api != NULL);
945d2e
-
945d2e
-    api->private = calloc(1, sizeof(struct controller_private));
945d2e
-    CRM_ASSERT(api->private != NULL);
945d2e
-    private = api->private;
945d2e
-
945d2e
-    if (client_name == NULL) {
945d2e
-        client_name = crm_system_name? crm_system_name : "client";
945d2e
-    }
945d2e
-    private->client_name = strdup(client_name);
945d2e
-    CRM_ASSERT(private->client_name != NULL);
945d2e
-
945d2e
-    if (client_uuid == NULL) {
945d2e
-        private->client_uuid = crm_generate_uuid();
945d2e
-    } else {
945d2e
-        private->client_uuid = strdup(client_uuid);
945d2e
-    }
945d2e
-    CRM_ASSERT(private->client_uuid != NULL);
945d2e
-
945d2e
-    api->connect = controller_api_connect;
945d2e
-    api->disconnect = controller_api_disconnect;
945d2e
-    api->replies_expected = controller_api_replies_expected;
945d2e
-    api->reprobe = controller_api_reprobe;
945d2e
-    api->fail_resource = controller_api_fail_resource;
945d2e
-    api->refresh_resource = controller_api_refresh_resource;
945d2e
-    return api;
945d2e
-}
945d2e
-
945d2e
-void
945d2e
-pcmk_free_controld_api(pcmk_controld_api_t *api)
945d2e
-{
945d2e
-    if (api != NULL) {
945d2e
-        struct controller_private *private = api->private;
945d2e
-
945d2e
-        api->disconnect(api);
945d2e
-        free(private->client_name);
945d2e
-        free(private->client_uuid);
945d2e
-        free(api->private);
945d2e
-        free(api);
945d2e
-    }
945d2e
-}
945d2e
diff --git a/tools/crm_resource_controller.h b/tools/crm_resource_controller.h
945d2e
deleted file mode 100644
945d2e
index 50e20b4..0000000
945d2e
--- a/tools/crm_resource_controller.h
945d2e
+++ /dev/null
945d2e
@@ -1,198 +0,0 @@
945d2e
-/*
945d2e
- * Copyright 2020 the Pacemaker project contributors
945d2e
- *
945d2e
- * The version control history for this file may have further details.
945d2e
- *
945d2e
- * This source code is licensed under the GNU Lesser General Public License
945d2e
- * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
945d2e
- */
945d2e
-#ifndef PCMK__CONTROLD_API_H
945d2e
-#define PCMK__CONTROLD_API_H
945d2e
-
945d2e
-#ifdef __cplusplus
945d2e
-extern "C" {
945d2e
-#endif
945d2e
-
945d2e
-#include <stdbool.h>                // bool
945d2e
-
945d2e
-/* This is a demonstration of an abstracted controller IPC API. It is expected
945d2e
- * that this will be improved and moved to libcrmcommon.
945d2e
- *
945d2e
- * @TODO We could consider whether it's reasonable to have a single type for
945d2e
- * all daemons' IPC APIs (e.g. pcmk_ipc_api_t instead of pcmk_*_api_t). They
945d2e
- * could potentially have common connect/disconnect methods and then a void* to
945d2e
- * a group of API-specific methods.
945d2e
- *
945d2e
- * In that case, the callback type would also need to be generic, taking
945d2e
- * (pcmk_ipc_api_t *api, void *api_data, void *user_data), with individual APIs
945d2e
- * having functions for getting useful info from api_data. If all APIs followed
945d2e
- * the call_id model, we could use int call_id instead of api_data.
945d2e
- *
945d2e
- * A major annoyance is that the controller IPC protocol currently does not have
945d2e
- * any way to tie a particular reply to a particular request. The current
945d2e
- * clients (crmadmin, crm_node, and crm_resource) simply know what kind of reply
945d2e
- * to expect for the kind of request they sent. In crm_resource's case, all it
945d2e
- * does is count replies, ignoring their content altogether.
945d2e
- *
945d2e
- * That really forces us to have a single callback for all events rather than a
945d2e
- * per-request callback. That in turn implies that callers can only provide a
945d2e
- * single user data pointer.
945d2e
- *
945d2e
- * @TODO Define protocol version constants to use in hello message.
945d2e
- * @TODO Allow callers to specify timeouts.
945d2e
- * @TODO Define call IDs for controller ops, while somehow maintaining backward
945d2e
- *       compatibility, since a client running on a Pacemaker Remote node could
945d2e
- *       be older or newer than the controller on the connection's cluster
945d2e
- *       node.
945d2e
- * @TODO The controller currently does not respond to hello messages. We should
945d2e
- *       establish a common connection handshake protocol for all daemons that
945d2e
- *       involves a hello message and acknowledgement. We should support sync
945d2e
- *       or async connection (i.e. block until the ack is received, or return
945d2e
- *       after the hello is sent and call a connection callback when the hello
945d2e
- *       ack is received).
945d2e
- */
945d2e
-
945d2e
-//! \internal
945d2e
-typedef struct pcmk_controld_api_s pcmk_controld_api_t;
945d2e
-
945d2e
-//! \internal
945d2e
-typedef struct pcmk_controld_api_callback_s {
945d2e
-    void (*callback)(pcmk_controld_api_t *api, void *api_data, void *user_data);
945d2e
-    void *user_data;
945d2e
-} pcmk_controld_api_cb_t;
945d2e
-
945d2e
-//! \internal
945d2e
-struct pcmk_controld_api_s {
945d2e
-    //! \internal
945d2e
-    void *private;
945d2e
-
945d2e
-    /*!
945d2e
-     * \internal
945d2e
-     * \brief Connect to the local controller
945d2e
-     *
945d2e
-     * \param[in] api           Controller API instance
945d2e
-     * \param[in] use_mainloop  If true, attach IPC to main loop
945d2e
-     * \param[in] dispatch_cb   If not NULL, call this when replies are received
945d2e
-     * \param[in] destroy_cb    If not NULL, call this if connection drops
945d2e
-     *
945d2e
-     * \return Standard Pacemaker return code
945d2e
-     * \note Only the pointers inside the callback objects need to be
945d2e
-     *       persistent, not the callback objects themselves. The destroy_cb
945d2e
-     *       will be called only for unrequested disconnects.
945d2e
-     */
945d2e
-    int (*connect)(pcmk_controld_api_t *api, bool use_mainloop,
945d2e
-                   pcmk_controld_api_cb_t *dispatch_cb,
945d2e
-                   pcmk_controld_api_cb_t *destroy_cb);
945d2e
-
945d2e
-    /*!
945d2e
-     * \internal
945d2e
-     * \brief Disconnect from the local controller
945d2e
-     *
945d2e
-     * \param[in] api       Controller API instance
945d2e
-     *
945d2e
-     * \return Standard Pacemaker return code
945d2e
-     */
945d2e
-    int (*disconnect)(pcmk_controld_api_t *api);
945d2e
-
945d2e
-    /*!
945d2e
-     * \internal
945d2e
-     * \brief Check number of replies still expected from controller
945d2e
-     *
945d2e
-     * \param[in] api       Controller API instance
945d2e
-     *
945d2e
-     * \return Number of expected replies
945d2e
-     */
945d2e
-    unsigned int (*replies_expected)(pcmk_controld_api_t *api);
945d2e
-
945d2e
-    /*!
945d2e
-     * \internal
945d2e
-     * \brief Send a reprobe controller operation
945d2e
-     *
945d2e
-     * \param[in] api          Controller API instance
945d2e
-     * \param[in] target_node  Name of node to reprobe
945d2e
-     * \param[in] router_node  Router node for host
945d2e
-     *
945d2e
-     * \return Standard Pacemaker return code
945d2e
-     */
945d2e
-    int (*reprobe)(pcmk_controld_api_t *api, const char *target_node,
945d2e
-                   const char *router_node);
945d2e
-
945d2e
-    /* @TODO These methods have a lot of arguments. One possibility would be to
945d2e
-     * make a struct for agent info (standard/provider/type), which theortically
945d2e
-     * could be used throughout pacemaker code. However that would end up being
945d2e
-     * really awkward to use generically, since sometimes you need to allocate
945d2e
-     * those strings (char *) and other times you only have references into XML
945d2e
-     * (const char *). We could make some structs just for this API.
945d2e
-     */
945d2e
-
945d2e
-    /*!
945d2e
-     * \internal
945d2e
-     * \brief Ask the controller to fail a resource
945d2e
-     *
945d2e
-     * \param[in] api          Controller API instance
945d2e
-     * \param[in] target_node  Name of node resource is on
945d2e
-     * \param[in] router_node  Router node for target
945d2e
-     * \param[in] rsc_id       ID of resource to fail
945d2e
-     * \param[in] rsc_long_id  Long ID of resource (if any)
945d2e
-     * \param[in] standard     Standard of resource
945d2e
-     * \param[in] provider     Provider of resource (if any)
945d2e
-     * \param[in] type         Type of resource to fail
945d2e
-     *
945d2e
-     * \return Standard Pacemaker return code
945d2e
-     */
945d2e
-    int (*fail_resource)(pcmk_controld_api_t *api, const char *target_node,
945d2e
-                         const char *router_node, const char *rsc_id,
945d2e
-                         const char *rsc_long_id, const char *standard,
945d2e
-                         const char *provider, const char *type);
945d2e
-
945d2e
-    /*!
945d2e
-     * \internal
945d2e
-     * \brief Ask the controller to refresh a resource
945d2e
-     *
945d2e
-     * \param[in] api          Controller API instance
945d2e
-     * \param[in] target_node  Name of node resource is on
945d2e
-     * \param[in] router_node  Router node for target
945d2e
-     * \param[in] rsc_id       ID of resource to refresh
945d2e
-     * \param[in] rsc_long_id  Long ID of resource (if any)
945d2e
-     * \param[in] standard     Standard of resource
945d2e
-     * \param[in] provider     Provider of resource (if any)
945d2e
-     * \param[in] type         Type of resource
945d2e
-     * \param[in] cib_only     If true, clean resource from CIB only
945d2e
-     *
945d2e
-     * \return Standard Pacemaker return code
945d2e
-     */
945d2e
-    int (*refresh_resource)(pcmk_controld_api_t *api, const char *target_node,
945d2e
-                            const char *router_node, const char *rsc_id,
945d2e
-                            const char *rsc_long_id, const char *standard,
945d2e
-                            const char *provider, const char *type,
945d2e
-                            bool cib_only);
945d2e
-};
945d2e
-
945d2e
-/*!
945d2e
- * \internal
945d2e
- * \brief Create new controller IPC API object for clients
945d2e
- *
945d2e
- * \param[in] client_name  Client name to use with IPC
945d2e
- * \param[in] client_uuid  Client UUID to use with IPC
945d2e
- *
945d2e
- * \return Newly allocated object
945d2e
- * \note This function asserts on errors, so it will never return NULL.
945d2e
- *       The caller is responsible for freeing the result with
945d2e
- *       pcmk_free_controld_api().
945d2e
- */
945d2e
-pcmk_controld_api_t *pcmk_new_controld_api(const char *client_name,
945d2e
-                                           const char *client_uuid);
945d2e
-
945d2e
-/*!
945d2e
- * \internal
945d2e
- * \brief Free a controller IPC API object
945d2e
- *
945d2e
- * \param[in] api  Controller IPC API object to free
945d2e
- */
945d2e
-void pcmk_free_controld_api(pcmk_controld_api_t *api);
945d2e
-
945d2e
-#ifdef __cplusplus
945d2e
-}
945d2e
-#endif
945d2e
-
945d2e
-#endif
945d2e
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
945d2e
index 37789d1..cc2abeb 100644
945d2e
--- a/tools/crm_resource_runtime.c
945d2e
+++ b/tools/crm_resource_runtime.c
945d2e
@@ -8,6 +8,7 @@
945d2e
  */
945d2e
 
945d2e
 #include <crm_resource.h>
945d2e
+#include <crm/common/ipc_controld.h>
945d2e
 
945d2e
 int resource_verbose = 0;
945d2e
 bool do_force = FALSE;
945d2e
@@ -460,7 +461,7 @@ cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
945d2e
 
945d2e
 // \return Standard Pacemaker return code
945d2e
 static int
945d2e
-send_lrm_rsc_op(pcmk_controld_api_t *controld_api, bool do_fail_resource,
945d2e
+send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
945d2e
                 const char *host_uname, const char *rsc_id,
945d2e
                 pe_working_set_t *data_set)
945d2e
 {
945d2e
@@ -528,14 +529,13 @@ send_lrm_rsc_op(pcmk_controld_api_t *controld_api, bool do_fail_resource,
945d2e
         rsc_api_id = rsc->id;
945d2e
     }
945d2e
     if (do_fail_resource) {
945d2e
-        return controld_api->fail_resource(controld_api, host_uname,
945d2e
-                                           router_node, rsc_api_id, rsc_long_id,
945d2e
-                                           rsc_class, rsc_provider, rsc_type);
945d2e
+        return pcmk_controld_api_fail(controld_api, host_uname, router_node,
945d2e
+                                      rsc_api_id, rsc_long_id,
945d2e
+                                      rsc_class, rsc_provider, rsc_type);
945d2e
     } else {
945d2e
-        return controld_api->refresh_resource(controld_api, host_uname,
945d2e
-                                              router_node, rsc_api_id,
945d2e
-                                              rsc_long_id, rsc_class,
945d2e
-                                              rsc_provider, rsc_type, cib_only);
945d2e
+        return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
945d2e
+                                         rsc_api_id, rsc_long_id, rsc_class,
945d2e
+                                         rsc_provider, rsc_type, cib_only);
945d2e
     }
945d2e
 }
945d2e
 
945d2e
@@ -558,7 +558,7 @@ rsc_fail_name(pe_resource_t *rsc)
945d2e
 
945d2e
 // \return Standard Pacemaker return code
945d2e
 static int
945d2e
-clear_rsc_history(pcmk_controld_api_t *controld_api, const char *host_uname,
945d2e
+clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
945d2e
                   const char *rsc_id, pe_working_set_t *data_set)
945d2e
 {
945d2e
     int rc = pcmk_ok;
945d2e
@@ -574,16 +574,16 @@ clear_rsc_history(pcmk_controld_api_t *controld_api, const char *host_uname,
945d2e
     }
945d2e
 
945d2e
     crm_trace("Processing %d mainloop inputs",
945d2e
-              controld_api->replies_expected(controld_api));
945d2e
+              pcmk_controld_api_replies_expected(controld_api));
945d2e
     while (g_main_context_iteration(NULL, FALSE)) {
945d2e
         crm_trace("Processed mainloop input, %d still remaining",
945d2e
-                  controld_api->replies_expected(controld_api));
945d2e
+                  pcmk_controld_api_replies_expected(controld_api));
945d2e
     }
945d2e
     return rc;
945d2e
 }
945d2e
 
945d2e
 static int
945d2e
-clear_rsc_failures(pcmk_controld_api_t *controld_api, const char *node_name,
945d2e
+clear_rsc_failures(pcmk_ipc_api_t *controld_api, const char *node_name,
945d2e
                    const char *rsc_id, const char *operation,
945d2e
                    const char *interval_spec, pe_working_set_t *data_set)
945d2e
 {
945d2e
@@ -683,7 +683,7 @@ clear_rsc_fail_attrs(pe_resource_t *rsc, const char *operation,
945d2e
 }
945d2e
 
945d2e
 int
945d2e
-cli_resource_delete(pcmk_controld_api_t *controld_api, const char *host_uname,
945d2e
+cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
945d2e
                     pe_resource_t *rsc, const char *operation,
945d2e
                     const char *interval_spec, bool just_failures,
945d2e
                     pe_working_set_t *data_set)
945d2e
@@ -792,7 +792,7 @@ cli_resource_delete(pcmk_controld_api_t *controld_api, const char *host_uname,
945d2e
 }
945d2e
 
945d2e
 int
945d2e
-cli_cleanup_all(pcmk_controld_api_t *controld_api, const char *node_name,
945d2e
+cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
945d2e
                 const char *operation, const char *interval_spec,
945d2e
                 pe_working_set_t *data_set)
945d2e
 {
945d2e
@@ -905,7 +905,7 @@ cli_resource_check(cib_t * cib_conn, pe_resource_t *rsc)
945d2e
 
945d2e
 // \return Standard Pacemaker return code
945d2e
 int
945d2e
-cli_resource_fail(pcmk_controld_api_t *controld_api, const char *host_uname,
945d2e
+cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
945d2e
                   const char *rsc_id, pe_working_set_t *data_set)
945d2e
 {
945d2e
     crm_notice("Failing %s on %s", rsc_id, host_uname);
945d2e
-- 
945d2e
1.8.3.1
945d2e
945d2e
945d2e
From ae14fa4a831e45eae0d78b0f42765fcf40c4ce56 Mon Sep 17 00:00:00 2001
945d2e
From: Ken Gaillot <kgaillot@redhat.com>
945d2e
Date: Tue, 14 Apr 2020 14:06:02 -0500
945d2e
Subject: [PATCH 4/6] Refactor: tools: convert crm_node to use new controller
945d2e
 IPC model
945d2e
945d2e
---
945d2e
 tools/crm_node.c | 281 +++++++++++++++++++++++++++----------------------------
945d2e
 1 file changed, 140 insertions(+), 141 deletions(-)
945d2e
945d2e
diff --git a/tools/crm_node.c b/tools/crm_node.c
945d2e
index db31f20..1773a36 100644
945d2e
--- a/tools/crm_node.c
945d2e
+++ b/tools/crm_node.c
945d2e
@@ -19,6 +19,7 @@
945d2e
 #include <crm/common/mainloop.h>
945d2e
 #include <crm/msg_xml.h>
945d2e
 #include <crm/cib.h>
945d2e
+#include <crm/common/ipc_controld.h>
945d2e
 #include <crm/common/attrd_internal.h>
945d2e
 
945d2e
 #define SUMMARY "crm_node - Tool for displaying low-level node information"
945d2e
@@ -39,7 +40,6 @@ gboolean command_cb(const gchar *option_name, const gchar *optarg, gpointer data
945d2e
 gboolean name_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
945d2e
 gboolean remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
945d2e
 
945d2e
-static char *pid_s = NULL;
945d2e
 static GMainLoop *mainloop = NULL;
945d2e
 static crm_exit_t exit_code = CRM_EX_OK;
945d2e
 
945d2e
@@ -140,11 +140,6 @@ remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError *
945d2e
 static void
945d2e
 crm_node_exit(crm_exit_t value)
945d2e
 {
945d2e
-    if (pid_s) {
945d2e
-        free(pid_s);
945d2e
-        pid_s = NULL;
945d2e
-    }
945d2e
-
945d2e
     exit_code = value;
945d2e
 
945d2e
     if (mainloop && g_main_loop_is_running(mainloop)) {
945d2e
@@ -155,173 +150,123 @@ crm_node_exit(crm_exit_t value)
945d2e
 }
945d2e
 
945d2e
 static void
945d2e
-exit_disconnect(gpointer user_data)
945d2e
-{
945d2e
-    fprintf(stderr, "error: Lost connection to cluster\n");
945d2e
-    crm_node_exit(CRM_EX_DISCONNECT);
945d2e
-}
945d2e
-
945d2e
-typedef int (*ipc_dispatch_fn) (const char *buffer, ssize_t length,
945d2e
-                                gpointer userdata);
945d2e
-
945d2e
-static crm_ipc_t *
945d2e
-new_mainloop_for_ipc(const char *system, ipc_dispatch_fn dispatch)
945d2e
+controller_event_cb(pcmk_ipc_api_t *controld_api,
945d2e
+                    enum pcmk_ipc_event event_type, crm_exit_t status,
945d2e
+                    void *event_data, void *user_data)
945d2e
 {
945d2e
-    mainloop_io_t *source = NULL;
945d2e
-    crm_ipc_t *ipc = NULL;
945d2e
-
945d2e
-    struct ipc_client_callbacks ipc_callbacks = {
945d2e
-        .dispatch = dispatch,
945d2e
-        .destroy = exit_disconnect
945d2e
-    };
945d2e
+    pcmk_controld_api_reply_t *reply = event_data;
945d2e
 
945d2e
-    mainloop = g_main_loop_new(NULL, FALSE);
945d2e
-    source = mainloop_add_ipc_client(system, G_PRIORITY_DEFAULT, 0,
945d2e
-                                     NULL, &ipc_callbacks);
945d2e
-    ipc = mainloop_get_ipc_client(source);
945d2e
-    if (ipc == NULL) {
945d2e
-        fprintf(stderr,
945d2e
-                "error: Could not connect to cluster (is it running?)\n");
945d2e
-        crm_node_exit(CRM_EX_DISCONNECT);
945d2e
-    }
945d2e
-    return ipc;
945d2e
-}
945d2e
-
945d2e
-static int
945d2e
-send_controller_hello(crm_ipc_t *controller)
945d2e
-{
945d2e
-    xmlNode *hello = NULL;
945d2e
-    int rc;
945d2e
-
945d2e
-    pid_s = pcmk__getpid_s();
945d2e
-    hello = create_hello_message(pid_s, crm_system_name, "1", "0");
945d2e
-    rc = crm_ipc_send(controller, hello, 0, 0, NULL);
945d2e
-    free_xml(hello);
945d2e
-    return (rc < 0)? rc : 0;
945d2e
-}
945d2e
-
945d2e
-static int
945d2e
-send_node_info_request(crm_ipc_t *controller, uint32_t nodeid)
945d2e
-{
945d2e
-    xmlNode *ping = NULL;
945d2e
-    int rc;
945d2e
-
945d2e
-    ping = create_request(CRM_OP_NODE_INFO, NULL, NULL, CRM_SYSTEM_CRMD,
945d2e
-                          crm_system_name, pid_s);
945d2e
-    if (nodeid > 0) {
945d2e
-        crm_xml_add_int(ping, XML_ATTR_ID, nodeid);
945d2e
-    }
945d2e
-    rc = crm_ipc_send(controller, ping, 0, 0, NULL);
945d2e
-    free_xml(ping);
945d2e
-    return (rc < 0)? rc : 0;
945d2e
-}
945d2e
+    switch (event_type) {
945d2e
+        case pcmk_ipc_event_disconnect:
945d2e
+            if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
945d2e
+                fprintf(stderr, "error: Lost connection to controller\n");
945d2e
+            }
945d2e
+            goto done;
945d2e
+            break;
945d2e
 
945d2e
-static int
945d2e
-dispatch_controller(const char *buffer, ssize_t length, gpointer userdata)
945d2e
-{
945d2e
-    xmlNode *message = string2xml(buffer);
945d2e
-    xmlNode *data = NULL;
945d2e
-    const char *value = NULL;
945d2e
+        case pcmk_ipc_event_reply:
945d2e
+            break;
945d2e
 
945d2e
-    if (message == NULL) {
945d2e
-        fprintf(stderr, "error: Could not understand reply from controller\n");
945d2e
-        crm_node_exit(CRM_EX_PROTOCOL);
945d2e
-        return 0;
945d2e
+        default:
945d2e
+            return;
945d2e
     }
945d2e
-    crm_log_xml_trace(message, "controller reply");
945d2e
-
945d2e
-    exit_code = CRM_EX_PROTOCOL;
945d2e
 
945d2e
-    // Validate reply
945d2e
-    value = crm_element_value(message, F_CRM_MSG_TYPE);
945d2e
-    if (safe_str_neq(value, XML_ATTR_RESPONSE)) {
945d2e
-        fprintf(stderr, "error: Message from controller was not a reply\n");
945d2e
+    if (status != CRM_EX_OK) {
945d2e
+        fprintf(stderr, "error: Bad reply from controller: %s\n",
945d2e
+                crm_exit_str(status));
945d2e
         goto done;
945d2e
     }
945d2e
-    value = crm_element_value(message, XML_ATTR_REFERENCE);
945d2e
-    if (value == NULL) {
945d2e
-        fprintf(stderr, "error: Controller reply did not specify original message\n");
945d2e
-        goto done;
945d2e
-    }
945d2e
-    data = get_message_xml(message, F_CRM_DATA);
945d2e
-    if (data == NULL) {
945d2e
-        fprintf(stderr, "error: Controller reply did not contain any data\n");
945d2e
+    if (reply->reply_type != pcmk_controld_reply_info) {
945d2e
+        fprintf(stderr, "error: Unknown reply type %d from controller\n",
945d2e
+                reply->reply_type);
945d2e
         goto done;
945d2e
     }
945d2e
 
945d2e
+    // Parse desired info from reply and display to user
945d2e
     switch (options.command) {
945d2e
         case 'i':
945d2e
-            value = crm_element_value(data, XML_ATTR_ID);
945d2e
-            if (value == NULL) {
945d2e
-                fprintf(stderr, "error: Controller reply did not contain node ID\n");
945d2e
-            } else {
945d2e
-                printf("%s\n", value);
945d2e
-                exit_code = CRM_EX_OK;
945d2e
+            if (reply->data.node_info.id == 0) {
945d2e
+                fprintf(stderr,
945d2e
+                        "error: Controller reply did not contain node ID\n");
945d2e
+                exit_code = CRM_EX_PROTOCOL;
945d2e
+                goto done;
945d2e
             }
945d2e
+            printf("%d\n", reply->data.node_info.id);
945d2e
             break;
945d2e
 
945d2e
         case 'n':
945d2e
         case 'N':
945d2e
-            value = crm_element_value(data, XML_ATTR_UNAME);
945d2e
-            if (value == NULL) {
945d2e
+            if (reply->data.node_info.uname == NULL) {
945d2e
                 fprintf(stderr, "Node is not known to cluster\n");
945d2e
                 exit_code = CRM_EX_NOHOST;
945d2e
-            } else {
945d2e
-                printf("%s\n", value);
945d2e
-                exit_code = CRM_EX_OK;
945d2e
+                goto done;
945d2e
             }
945d2e
+            printf("%s\n", reply->data.node_info.uname);
945d2e
             break;
945d2e
 
945d2e
         case 'q':
945d2e
-            value = crm_element_value(data, XML_ATTR_HAVE_QUORUM);
945d2e
-            if (value == NULL) {
945d2e
-                fprintf(stderr, "error: Controller reply did not contain quorum status\n");
945d2e
-            } else {
945d2e
-                bool quorum = crm_is_true(value);
945d2e
-
945d2e
-                printf("%d\n", quorum);
945d2e
-                exit_code = quorum? CRM_EX_OK : CRM_EX_QUORUM;
945d2e
+            printf("%d\n", reply->data.node_info.have_quorum);
945d2e
+            if (!(reply->data.node_info.have_quorum)) {
945d2e
+                exit_code = CRM_EX_QUORUM;
945d2e
+                goto done;
945d2e
             }
945d2e
             break;
945d2e
 
945d2e
         default:
945d2e
             fprintf(stderr, "internal error: Controller reply not expected\n");
945d2e
             exit_code = CRM_EX_SOFTWARE;
945d2e
-            break;
945d2e
+            goto done;
945d2e
     }
945d2e
 
945d2e
+    // Success
945d2e
+    exit_code = CRM_EX_OK;
945d2e
 done:
945d2e
-    free_xml(message);
945d2e
-    crm_node_exit(exit_code);
945d2e
-    return 0;
945d2e
+    pcmk_disconnect_ipc(controld_api);
945d2e
+    pcmk_quit_main_loop(mainloop, 10);
945d2e
 }
945d2e
 
945d2e
 static void
945d2e
 run_controller_mainloop(uint32_t nodeid)
945d2e
 {
945d2e
-    crm_ipc_t *controller = NULL;
945d2e
+    pcmk_ipc_api_t *controld_api = NULL;
945d2e
     int rc;
945d2e
 
945d2e
-    controller = new_mainloop_for_ipc(CRM_SYSTEM_CRMD, dispatch_controller);
945d2e
+    // Set disconnect exit code to handle unexpected disconnects
945d2e
+    exit_code = CRM_EX_DISCONNECT;
945d2e
+
945d2e
+    // Create controller IPC object
945d2e
+    rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
+        fprintf(stderr, "error: Could not connect to controller: %s\n",
945d2e
+                pcmk_rc_str(rc));
945d2e
+        return;
945d2e
+    }
945d2e
+    pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL);
945d2e
 
945d2e
-    rc = send_controller_hello(controller);
945d2e
-    if (rc < 0) {
945d2e
-        fprintf(stderr, "error: Could not register with controller: %s\n",
945d2e
-                pcmk_strerror(rc));
945d2e
-        crm_node_exit(crm_errno2exit(rc));
945d2e
+    // Connect to controller
945d2e
+    rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main);
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
+        fprintf(stderr, "error: Could not connect to controller: %s\n",
945d2e
+                pcmk_rc_str(rc));
945d2e
+        exit_code = pcmk_rc2exitc(rc);
945d2e
+        return;
945d2e
     }
945d2e
 
945d2e
-    rc = send_node_info_request(controller, nodeid);
945d2e
-    if (rc < 0) {
945d2e
+    rc = pcmk_controld_api_node_info(controld_api, nodeid);
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
         fprintf(stderr, "error: Could not ping controller: %s\n",
945d2e
-                pcmk_strerror(rc));
945d2e
-        crm_node_exit(crm_errno2exit(rc));
945d2e
+                pcmk_rc_str(rc));
945d2e
+        pcmk_disconnect_ipc(controld_api);
945d2e
+        exit_code = pcmk_rc2exitc(rc);
945d2e
+        return;
945d2e
     }
945d2e
 
945d2e
-    // Run main loop to get controller reply via dispatch_controller()
945d2e
+    // Run main loop to get controller reply via controller_event_cb()
945d2e
+    mainloop = g_main_loop_new(NULL, FALSE);
945d2e
     g_main_loop_run(mainloop);
945d2e
     g_main_loop_unref(mainloop);
945d2e
     mainloop = NULL;
945d2e
+    pcmk_free_ipc_api(controld_api);
945d2e
 }
945d2e
 
945d2e
 static void
945d2e
@@ -385,32 +330,56 @@ cib_remove_node(long id, const char *name)
945d2e
 }
945d2e
 
945d2e
 static int
945d2e
+controller_remove_node(const char *node_name, long nodeid)
945d2e
+{
945d2e
+    pcmk_ipc_api_t *controld_api = NULL;
945d2e
+    int rc;
945d2e
+
945d2e
+    // Create controller IPC object
945d2e
+    rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
+        fprintf(stderr, "error: Could not connect to controller: %s\n",
945d2e
+                pcmk_rc_str(rc));
945d2e
+        return ENOTCONN;
945d2e
+    }
945d2e
+
945d2e
+    // Connect to controller (without main loop)
945d2e
+    rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_sync);
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
+        fprintf(stderr, "error: Could not connect to controller: %s\n",
945d2e
+                pcmk_rc_str(rc));
945d2e
+        pcmk_free_ipc_api(controld_api);
945d2e
+        return rc;
945d2e
+    }
945d2e
+
945d2e
+    rc = pcmk_ipc_purge_node(controld_api, node_name, nodeid);
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
+        fprintf(stderr,
945d2e
+                "error: Could not clear node from controller's cache: %s\n",
945d2e
+                pcmk_rc_str(rc));
945d2e
+    }
945d2e
+
945d2e
+    pcmk_free_ipc_api(controld_api);
945d2e
+    return pcmk_rc_ok;
945d2e
+}
945d2e
+
945d2e
+static int
945d2e
 tools_remove_node_cache(const char *node_name, long nodeid, const char *target)
945d2e
 {
945d2e
     int rc = -1;
945d2e
-    crm_ipc_t *conn = crm_ipc_new(target, 0);
945d2e
+    crm_ipc_t *conn = NULL;
945d2e
     xmlNode *cmd = NULL;
945d2e
 
945d2e
+    conn = crm_ipc_new(target, 0);
945d2e
     if (!conn) {
945d2e
         return -ENOTCONN;
945d2e
     }
945d2e
-
945d2e
     if (!crm_ipc_connect(conn)) {
945d2e
         crm_perror(LOG_ERR, "Connection to %s failed", target);
945d2e
         crm_ipc_destroy(conn);
945d2e
         return -ENOTCONN;
945d2e
     }
945d2e
 
945d2e
-    if(safe_str_eq(target, CRM_SYSTEM_CRMD)) {
945d2e
-        // The controller requires a hello message before sending a request
945d2e
-        rc = send_controller_hello(conn);
945d2e
-        if (rc < 0) {
945d2e
-            fprintf(stderr, "error: Could not register with controller: %s\n",
945d2e
-                    pcmk_strerror(rc));
945d2e
-            return rc;
945d2e
-        }
945d2e
-    }
945d2e
-
945d2e
     crm_trace("Removing %s[%ld] from the %s membership cache",
945d2e
               node_name, nodeid, target);
945d2e
 
945d2e
@@ -427,9 +396,9 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target)
945d2e
             crm_xml_add_int(cmd, PCMK__XA_ATTR_NODE_ID, (int) nodeid);
945d2e
         }
945d2e
 
945d2e
-    } else {
945d2e
-        cmd = create_request(CRM_OP_RM_NODE_CACHE,
945d2e
-                             NULL, NULL, target, crm_system_name, pid_s);
945d2e
+    } else { // Fencer or pacemakerd
945d2e
+        cmd = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, target,
945d2e
+                             crm_system_name, NULL);
945d2e
         if (nodeid > 0) {
945d2e
             crm_xml_set_id(cmd, "%ld", nodeid);
945d2e
         }
945d2e
@@ -441,6 +410,7 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target)
945d2e
               target, node_name, nodeid, rc);
945d2e
 
945d2e
     if (rc > 0) {
945d2e
+        // @TODO Should this be done just once after all the rest?
945d2e
         rc = cib_remove_node(nodeid, node_name);
945d2e
     }
945d2e
 
945d2e
@@ -455,12 +425,12 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target)
945d2e
 static void
945d2e
 remove_node(const char *target_uname)
945d2e
 {
945d2e
+    int rc;
945d2e
     int d = 0;
945d2e
     long nodeid = 0;
945d2e
     const char *node_name = NULL;
945d2e
     char *endptr = NULL;
945d2e
     const char *daemons[] = {
945d2e
-        CRM_SYSTEM_CRMD,
945d2e
         "stonith-ng",
945d2e
         T_ATTRD,
945d2e
         CRM_SYSTEM_MCP,
945d2e
@@ -476,6 +446,12 @@ remove_node(const char *target_uname)
945d2e
         node_name = target_uname;
945d2e
     }
945d2e
 
945d2e
+    rc = controller_remove_node(node_name, nodeid);
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
+        exit_code = pcmk_rc2exitc(rc);
945d2e
+        return;
945d2e
+    }
945d2e
+
945d2e
     for (d = 0; d < DIMOF(daemons); d++) {
945d2e
         if (tools_remove_node_cache(node_name, nodeid, daemons[d])) {
945d2e
             crm_err("Failed to connect to %s to remove node '%s'",
945d2e
@@ -545,12 +521,34 @@ node_mcp_dispatch(const char *buffer, ssize_t length, gpointer userdata)
945d2e
 }
945d2e
 
945d2e
 static void
945d2e
+lost_pacemakerd(gpointer user_data)
945d2e
+{
945d2e
+    fprintf(stderr, "error: Lost connection to cluster\n");
945d2e
+    exit_code = CRM_EX_DISCONNECT;
945d2e
+    g_main_loop_quit(mainloop);
945d2e
+}
945d2e
+
945d2e
+static void
945d2e
 run_pacemakerd_mainloop(void)
945d2e
 {
945d2e
     crm_ipc_t *ipc = NULL;
945d2e
     xmlNode *poke = NULL;
945d2e
+    mainloop_io_t *source = NULL;
945d2e
 
945d2e
-    ipc = new_mainloop_for_ipc(CRM_SYSTEM_MCP, node_mcp_dispatch);
945d2e
+    struct ipc_client_callbacks ipc_callbacks = {
945d2e
+        .dispatch = node_mcp_dispatch,
945d2e
+        .destroy = lost_pacemakerd
945d2e
+    };
945d2e
+
945d2e
+    source = mainloop_add_ipc_client(CRM_SYSTEM_MCP, G_PRIORITY_DEFAULT, 0,
945d2e
+                                     NULL, &ipc_callbacks);
945d2e
+    ipc = mainloop_get_ipc_client(source);
945d2e
+    if (ipc == NULL) {
945d2e
+        fprintf(stderr,
945d2e
+                "error: Could not connect to cluster (is it running?)\n");
945d2e
+        exit_code = CRM_EX_DISCONNECT;
945d2e
+        return;
945d2e
+    }
945d2e
 
945d2e
     // Sending anything will get us a list of nodes
945d2e
     poke = create_xml_node(NULL, "poke");
945d2e
@@ -558,6 +556,7 @@ run_pacemakerd_mainloop(void)
945d2e
     free_xml(poke);
945d2e
 
945d2e
     // Handle reply via node_mcp_dispatch()
945d2e
+    mainloop = g_main_loop_new(NULL, FALSE);
945d2e
     g_main_loop_run(mainloop);
945d2e
     g_main_loop_unref(mainloop);
945d2e
     mainloop = NULL;
945d2e
-- 
945d2e
1.8.3.1
945d2e
945d2e
945d2e
From a361f764cb28630d440eec0f3e04a4f3812825eb Mon Sep 17 00:00:00 2001
945d2e
From: Ken Gaillot <kgaillot@redhat.com>
945d2e
Date: Tue, 14 Apr 2020 16:05:15 -0500
945d2e
Subject: [PATCH 5/6] Refactor: tools: remove dead code from crm_node
945d2e
945d2e
---
945d2e
 tools/crm_node.c | 22 +---------------------
945d2e
 1 file changed, 1 insertion(+), 21 deletions(-)
945d2e
945d2e
diff --git a/tools/crm_node.c b/tools/crm_node.c
945d2e
index 1773a36..57c2ee5 100644
945d2e
--- a/tools/crm_node.c
945d2e
+++ b/tools/crm_node.c
945d2e
@@ -130,25 +130,6 @@ remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError *
945d2e
     return TRUE;
945d2e
 }
945d2e
 
945d2e
-/*!
945d2e
- * \internal
945d2e
- * \brief Exit crm_node
945d2e
- * Clean up memory, and either quit mainloop (if running) or exit
945d2e
- *
945d2e
- * \param[in] value  Exit status
945d2e
- */
945d2e
-static void
945d2e
-crm_node_exit(crm_exit_t value)
945d2e
-{
945d2e
-    exit_code = value;
945d2e
-
945d2e
-    if (mainloop && g_main_loop_is_running(mainloop)) {
945d2e
-        g_main_loop_quit(mainloop);
945d2e
-    } else {
945d2e
-        crm_exit(exit_code);
945d2e
-    }
945d2e
-}
945d2e
-
945d2e
 static void
945d2e
 controller_event_cb(pcmk_ipc_api_t *controld_api,
945d2e
                     enum pcmk_ipc_event event_type, crm_exit_t status,
945d2e
@@ -660,6 +641,5 @@ done:
945d2e
     g_strfreev(processed_args);
945d2e
     g_clear_error(&error);
945d2e
     pcmk__free_arg_context(context);
945d2e
-    crm_node_exit(exit_code);
945d2e
-    return exit_code;
945d2e
+    return crm_exit(exit_code);
945d2e
 }
945d2e
-- 
945d2e
1.8.3.1
945d2e
945d2e
945d2e
From 591944539259f6294450517770d95c7a02fc599c Mon Sep 17 00:00:00 2001
945d2e
From: Ken Gaillot <kgaillot@redhat.com>
945d2e
Date: Mon, 20 Apr 2020 15:48:15 -0500
945d2e
Subject: [PATCH 6/6] Refactor: tools: convert crmadmin to use new controller
945d2e
 IPC model
945d2e
945d2e
---
945d2e
 tools/crmadmin.c | 484 ++++++++++++++++++++++++-------------------------------
945d2e
 1 file changed, 208 insertions(+), 276 deletions(-)
945d2e
945d2e
diff --git a/tools/crmadmin.c b/tools/crmadmin.c
945d2e
index 3e9e959..4688458 100644
945d2e
--- a/tools/crmadmin.c
945d2e
+++ b/tools/crmadmin.c
945d2e
@@ -9,56 +9,44 @@
945d2e
 
945d2e
 #include <crm_internal.h>
945d2e
 
945d2e
-#include <sys/param.h>
945d2e
-
945d2e
 #include <stdio.h>
945d2e
-#include <sys/types.h>
945d2e
-#include <unistd.h>
945d2e
+#include <stdbool.h>
945d2e
+#include <stdlib.h>             // atoi()
945d2e
 
945d2e
-#include <stdlib.h>
945d2e
-#include <errno.h>
945d2e
-#include <fcntl.h>
945d2e
-#include <libgen.h>
945d2e
 #include <glib.h>               // gboolean, GMainLoop, etc.
945d2e
+#include <libxml/tree.h>        // xmlNode
945d2e
 
945d2e
 #include <crm/crm.h>
945d2e
+#include <crm/cib.h>
945d2e
 #include <crm/msg_xml.h>
945d2e
 #include <crm/common/xml.h>
945d2e
-
945d2e
+#include <crm/common/ipc_controld.h>
945d2e
 #include <crm/common/mainloop.h>
945d2e
 
945d2e
-#include <crm/cib.h>
945d2e
-
945d2e
 #define DEFAULT_MESSAGE_TIMEOUT_MS 30000
945d2e
 
945d2e
 static guint message_timer_id = 0;
945d2e
 static guint message_timeout_ms = DEFAULT_MESSAGE_TIMEOUT_MS;
945d2e
 static GMainLoop *mainloop = NULL;
945d2e
-static crm_ipc_t *crmd_channel = NULL;
945d2e
-static char *admin_uuid = NULL;
945d2e
-
945d2e
-gboolean do_init(void);
945d2e
-int do_work(void);
945d2e
-void crmadmin_ipc_connection_destroy(gpointer user_data);
945d2e
-int admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata);
945d2e
-int do_find_node_list(xmlNode * xml_node);
945d2e
+
945d2e
+bool do_work(pcmk_ipc_api_t *api);
945d2e
+void do_find_node_list(xmlNode *xml_node);
945d2e
 gboolean admin_message_timeout(gpointer data);
945d2e
 
945d2e
+static enum {
945d2e
+    cmd_none,
945d2e
+    cmd_shutdown,
945d2e
+    cmd_health,
945d2e
+    cmd_elect_dc,
945d2e
+    cmd_whois_dc,
945d2e
+    cmd_list_nodes,
945d2e
+} command = cmd_none;
945d2e
+
945d2e
 static gboolean BE_VERBOSE = FALSE;
945d2e
-static int expected_responses = 1;
945d2e
 static gboolean BASH_EXPORT = FALSE;
945d2e
-static gboolean DO_HEALTH = FALSE;
945d2e
-static gboolean DO_RESET = FALSE;
945d2e
-static gboolean DO_RESOURCE = FALSE;
945d2e
-static gboolean DO_ELECT_DC = FALSE;
945d2e
-static gboolean DO_WHOIS_DC = FALSE;
945d2e
-static gboolean DO_NODE_LIST = FALSE;
945d2e
 static gboolean BE_SILENT = FALSE;
945d2e
-static gboolean DO_RESOURCE_LIST = FALSE;
945d2e
-static const char *crmd_operation = NULL;
945d2e
 static char *dest_node = NULL;
945d2e
 static crm_exit_t exit_code = CRM_EX_OK;
945d2e
-static const char *sys_to = NULL;
945d2e
 
945d2e
 static pcmk__cli_option_t long_options[] = {
945d2e
     // long option, argument type, storage, short option, description, flags
945d2e
@@ -133,7 +121,8 @@ static pcmk__cli_option_t long_options[] = {
945d2e
     },
945d2e
     {
945d2e
         "bash-export", no_argument, NULL, 'B',
945d2e
-        "Create Bash export entries of the form 'export uname=uuid'\n",
945d2e
+        "Display nodes as shell commands of the form 'export uname=uuid' "
945d2e
+            "(valid with -N/--nodes)'\n",
945d2e
         pcmk__option_default
945d2e
     },
945d2e
     {
945d2e
@@ -142,13 +131,98 @@ static pcmk__cli_option_t long_options[] = {
945d2e
     },
945d2e
     {
945d2e
         "-spacer-", no_argument, NULL, '-',
945d2e
-        " The -K and -E commands are rarely used and may be removed in "
945d2e
-            "future versions.",
945d2e
+        "The -K and -E commands do not work and may be removed in a future "
945d2e
+            "version.",
945d2e
         pcmk__option_default
945d2e
     },
945d2e
     { 0, 0, 0, 0 }
945d2e
 };
945d2e
 
945d2e
+static void
945d2e
+quit_main_loop(crm_exit_t ec)
945d2e
+{
945d2e
+    exit_code = ec;
945d2e
+    if (mainloop != NULL) {
945d2e
+        GMainLoop *mloop = mainloop;
945d2e
+
945d2e
+        mainloop = NULL; // Don't re-enter this block
945d2e
+        pcmk_quit_main_loop(mloop, 10);
945d2e
+        g_main_loop_unref(mloop);
945d2e
+    }
945d2e
+}
945d2e
+
945d2e
+static void
945d2e
+controller_event_cb(pcmk_ipc_api_t *controld_api,
945d2e
+                    enum pcmk_ipc_event event_type, crm_exit_t status,
945d2e
+                    void *event_data, void *user_data)
945d2e
+{
945d2e
+    pcmk_controld_api_reply_t *reply = event_data;
945d2e
+
945d2e
+    switch (event_type) {
945d2e
+        case pcmk_ipc_event_disconnect:
945d2e
+            if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
945d2e
+                fprintf(stderr, "error: Lost connection to controller\n");
945d2e
+            }
945d2e
+            goto done;
945d2e
+            break;
945d2e
+
945d2e
+        case pcmk_ipc_event_reply:
945d2e
+            break;
945d2e
+
945d2e
+        default:
945d2e
+            return;
945d2e
+    }
945d2e
+
945d2e
+    if (message_timer_id != 0) {
945d2e
+        g_source_remove(message_timer_id);
945d2e
+        message_timer_id = 0;
945d2e
+    }
945d2e
+
945d2e
+    if (status != CRM_EX_OK) {
945d2e
+        fprintf(stderr, "error: Bad reply from controller: %s",
945d2e
+                crm_exit_str(status));
945d2e
+        exit_code = status;
945d2e
+        goto done;
945d2e
+    }
945d2e
+
945d2e
+    if (reply->reply_type != pcmk_controld_reply_ping) {
945d2e
+        fprintf(stderr, "error: Unknown reply type %d from controller\n",
945d2e
+                reply->reply_type);
945d2e
+        goto done;
945d2e
+    }
945d2e
+
945d2e
+    // Parse desired information from reply
945d2e
+    switch (command) {
945d2e
+        case cmd_health:
945d2e
+            printf("Status of %s@%s: %s (%s)\n",
945d2e
+                   reply->data.ping.sys_from,
945d2e
+                   reply->host_from,
945d2e
+                   reply->data.ping.fsa_state,
945d2e
+                   reply->data.ping.result);
945d2e
+            if (BE_SILENT && (reply->data.ping.fsa_state != NULL)) {
945d2e
+                fprintf(stderr, "%s\n", reply->data.ping.fsa_state);
945d2e
+            }
945d2e
+            exit_code = CRM_EX_OK;
945d2e
+            break;
945d2e
+
945d2e
+        case cmd_whois_dc:
945d2e
+            printf("Designated Controller is: %s\n", reply->host_from);
945d2e
+            if (BE_SILENT && (reply->host_from != NULL)) {
945d2e
+                fprintf(stderr, "%s\n", reply->host_from);
945d2e
+            }
945d2e
+            exit_code = CRM_EX_OK;
945d2e
+            break;
945d2e
+
945d2e
+        default: // Not really possible here
945d2e
+            exit_code = CRM_EX_SOFTWARE;
945d2e
+            break;
945d2e
+    }
945d2e
+
945d2e
+done:
945d2e
+    pcmk_disconnect_ipc(controld_api);
945d2e
+    quit_main_loop(exit_code);
945d2e
+}
945d2e
+
945d2e
 // \return Standard Pacemaker return code
945d2e
 static int
945d2e
 list_nodes()
945d2e
@@ -181,6 +255,9 @@ main(int argc, char **argv)
945d2e
     int option_index = 0;
945d2e
     int argerr = 0;
945d2e
     int flag;
945d2e
+    int rc;
945d2e
+    pcmk_ipc_api_t *controld_api = NULL;
945d2e
+    bool need_controld_api = true;
945d2e
 
945d2e
     crm_log_cli_init("crmadmin");
945d2e
     pcmk__set_cli_options(NULL, "<command> [options]", long_options,
945d2e
@@ -211,33 +288,40 @@ main(int argc, char **argv)
945d2e
                 pcmk__cli_help(flag, CRM_EX_OK);
945d2e
                 break;
945d2e
             case 'D':
945d2e
-                DO_WHOIS_DC = TRUE;
945d2e
+                command = cmd_whois_dc;
945d2e
                 break;
945d2e
             case 'B':
945d2e
                 BASH_EXPORT = TRUE;
945d2e
                 break;
945d2e
             case 'K':
945d2e
-                DO_RESET = TRUE;
945d2e
+                command = cmd_shutdown;
945d2e
                 crm_trace("Option %c => %s", flag, optarg);
945d2e
+                if (dest_node != NULL) {
945d2e
+                    free(dest_node);
945d2e
+                }
945d2e
                 dest_node = strdup(optarg);
945d2e
-                crmd_operation = CRM_OP_LOCAL_SHUTDOWN;
945d2e
                 break;
945d2e
             case 'q':
945d2e
                 BE_SILENT = TRUE;
945d2e
                 break;
945d2e
             case 'S':
945d2e
-                DO_HEALTH = TRUE;
945d2e
+                command = cmd_health;
945d2e
                 crm_trace("Option %c => %s", flag, optarg);
945d2e
+                if (dest_node != NULL) {
945d2e
+                    free(dest_node);
945d2e
+                }
945d2e
                 dest_node = strdup(optarg);
945d2e
                 break;
945d2e
             case 'E':
945d2e
-                DO_ELECT_DC = TRUE;
945d2e
+                command = cmd_elect_dc;
945d2e
                 break;
945d2e
             case 'N':
945d2e
-                DO_NODE_LIST = TRUE;
945d2e
+                command = cmd_list_nodes;
945d2e
+                need_controld_api = false;
945d2e
                 break;
945d2e
             case 'H':
945d2e
-                DO_HEALTH = TRUE;
945d2e
+                fprintf(stderr, "Cluster-wide health option not supported\n");
945d2e
+                ++argerr;
945d2e
                 break;
945d2e
             default:
945d2e
                 printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
945d2e
@@ -257,285 +341,133 @@ main(int argc, char **argv)
945d2e
         ++argerr;
945d2e
     }
945d2e
 
945d2e
+    if (command == cmd_none) {
945d2e
+        fprintf(stderr, "error: Must specify a command option\n\n");
945d2e
+        ++argerr;
945d2e
+    }
945d2e
+
945d2e
     if (argerr) {
945d2e
         pcmk__cli_help('?', CRM_EX_USAGE);
945d2e
     }
945d2e
 
945d2e
-    if (do_init()) {
945d2e
-        int res = 0;
945d2e
-
945d2e
-        res = do_work();
945d2e
-        if (res > 0) {
945d2e
-            /* wait for the reply by creating a mainloop and running it until
945d2e
-             * the callbacks are invoked...
945d2e
-             */
945d2e
-            mainloop = g_main_loop_new(NULL, FALSE);
945d2e
-            crm_trace("Waiting for %d replies from the local CRM", expected_responses);
945d2e
-
945d2e
-            message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL);
945d2e
-
945d2e
-            g_main_loop_run(mainloop);
945d2e
-
945d2e
-        } else if (res < 0) {
945d2e
-            crm_err("No message to send");
945d2e
-            exit_code = CRM_EX_ERROR;
945d2e
+    // Connect to the controller if needed
945d2e
+    if (need_controld_api) {
945d2e
+        rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
945d2e
+        if (controld_api == NULL) {
945d2e
+            fprintf(stderr, "error: Could not connect to controller: %s\n",
945d2e
+                    pcmk_rc_str(rc));
945d2e
+            exit_code = pcmk_rc2exitc(rc);
945d2e
+            goto done;
945d2e
         }
945d2e
-    } else {
945d2e
-        crm_warn("Init failed, could not perform requested operations");
945d2e
-        exit_code = CRM_EX_UNAVAILABLE;
945d2e
-    }
945d2e
-
945d2e
-    crm_trace("%s exiting normally", crm_system_name);
945d2e
-    return exit_code;
945d2e
-}
945d2e
-
945d2e
-int
945d2e
-do_work(void)
945d2e
-{
945d2e
-    int ret = 1;
945d2e
-
945d2e
-    /* construct the request */
945d2e
-    xmlNode *msg_data = NULL;
945d2e
-    gboolean all_is_good = TRUE;
945d2e
-
945d2e
-    if (DO_HEALTH == TRUE) {
945d2e
-        crm_trace("Querying the system");
945d2e
-
945d2e
-        sys_to = CRM_SYSTEM_DC;
945d2e
-
945d2e
-        if (dest_node != NULL) {
945d2e
-            sys_to = CRM_SYSTEM_CRMD;
945d2e
-            crmd_operation = CRM_OP_PING;
945d2e
-
945d2e
-            if (BE_VERBOSE) {
945d2e
-                expected_responses = 1;
945d2e
-            }
945d2e
-
945d2e
-        } else {
945d2e
-            crm_info("Cluster-wide health not available yet");
945d2e
-            all_is_good = FALSE;
945d2e
+        pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL);
945d2e
+        rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main);
945d2e
+        if (rc != pcmk_rc_ok) {
945d2e
+            fprintf(stderr, "error: Could not connect to controller: %s\n",
945d2e
+                    pcmk_rc_str(rc));
945d2e
+            exit_code = pcmk_rc2exitc(rc);
945d2e
+            goto done;
945d2e
         }
945d2e
-
945d2e
-    } else if (DO_ELECT_DC) {
945d2e
-        /* tell the local node to initiate an election */
945d2e
-
945d2e
-        dest_node = NULL;
945d2e
-        sys_to = CRM_SYSTEM_CRMD;
945d2e
-        crmd_operation = CRM_OP_VOTE;
945d2e
-        ret = 0;                /* no return message */
945d2e
-
945d2e
-    } else if (DO_WHOIS_DC) {
945d2e
-        dest_node = NULL;
945d2e
-        sys_to = CRM_SYSTEM_DC;
945d2e
-        crmd_operation = CRM_OP_PING;
945d2e
-
945d2e
-    } else if (DO_NODE_LIST) {
945d2e
-        crm_exit(pcmk_rc2exitc(list_nodes()));
945d2e
-
945d2e
-    } else if (DO_RESET) {
945d2e
-        /* tell dest_node to initiate the shutdown procedure
945d2e
-         *
945d2e
-         * if dest_node is NULL, the request will be sent to the
945d2e
-         *   local node
945d2e
-         */
945d2e
-        sys_to = CRM_SYSTEM_CRMD;
945d2e
-        ret = 0;                /* no return message */
945d2e
-
945d2e
-    } else {
945d2e
-        crm_err("Unknown options");
945d2e
-        all_is_good = FALSE;
945d2e
     }
945d2e
 
945d2e
-    if (all_is_good == FALSE) {
945d2e
-        crm_err("Creation of request failed.  No message to send");
945d2e
-        return -1;
945d2e
+    if (do_work(controld_api)) {
945d2e
+        // A reply is needed from controller, so run main loop to get it
945d2e
+        exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
945d2e
+        mainloop = g_main_loop_new(NULL, FALSE);
945d2e
+        message_timer_id = g_timeout_add(message_timeout_ms,
945d2e
+                                         admin_message_timeout, NULL);
945d2e
+        g_main_loop_run(mainloop);
945d2e
     }
945d2e
 
945d2e
-/* send it */
945d2e
-    if (crmd_channel == NULL) {
945d2e
-        crm_err("The IPC connection is not valid, cannot send anything");
945d2e
-        return -1;
945d2e
-    }
945d2e
-
945d2e
-    if (sys_to == NULL) {
945d2e
-        if (dest_node != NULL) {
945d2e
-            sys_to = CRM_SYSTEM_CRMD;
945d2e
-        } else {
945d2e
-            sys_to = CRM_SYSTEM_DC;
945d2e
-        }
945d2e
-    }
945d2e
-
945d2e
-    {
945d2e
-        xmlNode *cmd = create_request(crmd_operation, msg_data, dest_node, sys_to,
945d2e
-                                      crm_system_name, admin_uuid);
945d2e
-
945d2e
-        crm_ipc_send(crmd_channel, cmd, 0, 0, NULL);
945d2e
-        free_xml(cmd);
945d2e
-    }
945d2e
-
945d2e
-    return ret;
945d2e
-}
945d2e
+done:
945d2e
+    if (controld_api != NULL) {
945d2e
+        pcmk_ipc_api_t *capi = controld_api;
945d2e
 
945d2e
-void
945d2e
-crmadmin_ipc_connection_destroy(gpointer user_data)
945d2e
-{
945d2e
-    crm_err("Connection to controller was terminated");
945d2e
-    if (mainloop) {
945d2e
-        g_main_loop_quit(mainloop);
945d2e
-    } else {
945d2e
-        crm_exit(CRM_EX_DISCONNECT);
945d2e
+        controld_api = NULL; // Ensure we can't free this twice
945d2e
+        pcmk_free_ipc_api(capi);
945d2e
     }
945d2e
-}
945d2e
-
945d2e
-struct ipc_client_callbacks crm_callbacks = {
945d2e
-    .dispatch = admin_msg_callback,
945d2e
-    .destroy = crmadmin_ipc_connection_destroy
945d2e
-};
945d2e
-
945d2e
-gboolean
945d2e
-do_init(void)
945d2e
-{
945d2e
-    mainloop_io_t *source =
945d2e
-        mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks);
945d2e
-
945d2e
-    admin_uuid = pcmk__getpid_s();
945d2e
-
945d2e
-    crmd_channel = mainloop_get_ipc_client(source);
945d2e
-
945d2e
-    if (DO_RESOURCE || DO_RESOURCE_LIST || DO_NODE_LIST) {
945d2e
-        return TRUE;
945d2e
-
945d2e
-    } else if (crmd_channel != NULL) {
945d2e
-        xmlNode *xml = create_hello_message(admin_uuid, crm_system_name, "0", "1");
945d2e
-
945d2e
-        crm_ipc_send(crmd_channel, xml, 0, 0, NULL);
945d2e
-        return TRUE;
945d2e
+    if (mainloop != NULL) {
945d2e
+        g_main_loop_unref(mainloop);
945d2e
+        mainloop = NULL;
945d2e
     }
945d2e
-    return FALSE;
945d2e
+    return crm_exit(exit_code);
945d2e
 }
945d2e
 
945d2e
-static bool
945d2e
-validate_crm_message(xmlNode * msg, const char *sys, const char *uuid, const char *msg_type)
945d2e
+// \return True if reply from controller is needed
945d2e
+bool
945d2e
+do_work(pcmk_ipc_api_t *controld_api)
945d2e
 {
945d2e
-    const char *type = NULL;
945d2e
-    const char *crm_msg_reference = NULL;
945d2e
-
945d2e
-    if (msg == NULL) {
945d2e
-        return FALSE;
945d2e
-    }
945d2e
+    bool need_reply = false;
945d2e
+    int rc = pcmk_rc_ok;
945d2e
 
945d2e
-    type = crm_element_value(msg, F_CRM_MSG_TYPE);
945d2e
-    crm_msg_reference = crm_element_value(msg, XML_ATTR_REFERENCE);
945d2e
-
945d2e
-    if (type == NULL) {
945d2e
-        crm_info("No message type defined.");
945d2e
-        return FALSE;
945d2e
-
945d2e
-    } else if (msg_type != NULL && strcasecmp(msg_type, type) != 0) {
945d2e
-        crm_info("Expecting a (%s) message but received a (%s).", msg_type, type);
945d2e
-        return FALSE;
945d2e
-    }
945d2e
-
945d2e
-    if (crm_msg_reference == NULL) {
945d2e
-        crm_info("No message crm_msg_reference defined.");
945d2e
-        return FALSE;
945d2e
-    }
945d2e
-
945d2e
-    return TRUE;
945d2e
-}
945d2e
-
945d2e
-int
945d2e
-admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata)
945d2e
-{
945d2e
-    static int received_responses = 0;
945d2e
-    xmlNode *xml = string2xml(buffer);
945d2e
-
945d2e
-    received_responses++;
945d2e
-    g_source_remove(message_timer_id);
945d2e
-
945d2e
-    crm_log_xml_trace(xml, "ipc");
945d2e
-
945d2e
-    if (xml == NULL) {
945d2e
-        crm_info("XML in IPC message was not valid... " "discarding.");
945d2e
-
945d2e
-    } else if (validate_crm_message(xml, crm_system_name, admin_uuid, XML_ATTR_RESPONSE) == FALSE) {
945d2e
-        crm_trace("Message was not a CRM response. Discarding.");
945d2e
-
945d2e
-    } else if (DO_HEALTH) {
945d2e
-        xmlNode *data = get_message_xml(xml, F_CRM_DATA);
945d2e
-        const char *state = crm_element_value(data, XML_PING_ATTR_CRMDSTATE);
945d2e
+    switch (command) {
945d2e
+        case cmd_shutdown:
945d2e
+            rc = pcmk_controld_api_shutdown(controld_api, dest_node);
945d2e
+            break;
945d2e
 
945d2e
-        printf("Status of %s@%s: %s (%s)\n",
945d2e
-               crm_element_value(data, XML_PING_ATTR_SYSFROM),
945d2e
-               crm_element_value(xml, F_CRM_HOST_FROM),
945d2e
-               state, crm_element_value(data, XML_PING_ATTR_STATUS));
945d2e
+        case cmd_health:    // dest_node != NULL
945d2e
+        case cmd_whois_dc:  // dest_node == NULL
945d2e
+            rc = pcmk_controld_api_ping(controld_api, dest_node);
945d2e
+            need_reply = true;
945d2e
+            break;
945d2e
 
945d2e
-        if (BE_SILENT && state != NULL) {
945d2e
-            fprintf(stderr, "%s\n", state);
945d2e
-        }
945d2e
+        case cmd_elect_dc:
945d2e
+            rc = pcmk_controld_api_start_election(controld_api);
945d2e
+            break;
945d2e
 
945d2e
-    } else if (DO_WHOIS_DC) {
945d2e
-        const char *dc = crm_element_value(xml, F_CRM_HOST_FROM);
945d2e
+        case cmd_list_nodes:
945d2e
+            rc = list_nodes();
945d2e
+            break;
945d2e
 
945d2e
-        printf("Designated Controller is: %s\n", dc);
945d2e
-        if (BE_SILENT && dc != NULL) {
945d2e
-            fprintf(stderr, "%s\n", dc);
945d2e
-        }
945d2e
-        crm_exit(CRM_EX_OK);
945d2e
+        case cmd_none: // not actually possible here
945d2e
+            break;
945d2e
     }
945d2e
-
945d2e
-    free_xml(xml);
945d2e
-
945d2e
-    if (received_responses >= expected_responses) {
945d2e
-        crm_trace("Received expected number (%d) of replies, exiting normally",
945d2e
-                   expected_responses);
945d2e
-        crm_exit(CRM_EX_OK);
945d2e
+    if (rc != pcmk_rc_ok) {
945d2e
+        fprintf(stderr, "error: Command failed: %s", pcmk_rc_str(rc));
945d2e
+        exit_code = pcmk_rc2exitc(rc);
945d2e
     }
945d2e
-
945d2e
-    message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL);
945d2e
-    return 0;
945d2e
+    return need_reply;
945d2e
 }
945d2e
 
945d2e
 gboolean
945d2e
 admin_message_timeout(gpointer data)
945d2e
 {
945d2e
-    fprintf(stderr, "No messages received in %d seconds.. aborting\n",
945d2e
-            (int)message_timeout_ms / 1000);
945d2e
-    crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000);
945d2e
-    exit_code = CRM_EX_TIMEOUT;
945d2e
-    g_main_loop_quit(mainloop);
945d2e
-    return FALSE;
945d2e
+    fprintf(stderr,
945d2e
+            "error: No reply received from controller before timeout (%dms)\n",
945d2e
+            message_timeout_ms);
945d2e
+    message_timer_id = 0;
945d2e
+    quit_main_loop(CRM_EX_TIMEOUT);
945d2e
+    return FALSE; // Tells glib to remove source
945d2e
 }
945d2e
 
945d2e
-int
945d2e
+void
945d2e
 do_find_node_list(xmlNode * xml_node)
945d2e
 {
945d2e
     int found = 0;
945d2e
     xmlNode *node = NULL;
945d2e
     xmlNode *nodes = get_object_root(XML_CIB_TAG_NODES, xml_node);
945d2e
 
945d2e
-    for (node = __xml_first_child_element(nodes); node != NULL;
945d2e
-         node = __xml_next_element(node)) {
945d2e
+    for (node = first_named_child(nodes, XML_CIB_TAG_NODE); node != NULL;
945d2e
+         node = crm_next_same_xml(node)) {
945d2e
 
945d2e
-        if (crm_str_eq((const char *)node->name, XML_CIB_TAG_NODE, TRUE)) {
945d2e
+        if (BASH_EXPORT) {
945d2e
+            printf("export %s=%s\n",
945d2e
+                   crm_element_value(node, XML_ATTR_UNAME),
945d2e
+                   crm_element_value(node, XML_ATTR_ID));
945d2e
+        } else {
945d2e
+            const char *node_type = crm_element_value(node, XML_ATTR_TYPE);
945d2e
 
945d2e
-            if (BASH_EXPORT) {
945d2e
-                printf("export %s=%s\n",
945d2e
-                       crm_element_value(node, XML_ATTR_UNAME),
945d2e
-                       crm_element_value(node, XML_ATTR_ID));
945d2e
-            } else {
945d2e
-                printf("%s node: %s (%s)\n",
945d2e
-                       crm_element_value(node, XML_ATTR_TYPE),
945d2e
-                       crm_element_value(node, XML_ATTR_UNAME),
945d2e
-                       crm_element_value(node, XML_ATTR_ID));
945d2e
+            if (node_type == NULL) {
945d2e
+                node_type = "member";
945d2e
             }
945d2e
-            found++;
945d2e
+            printf("%s node: %s (%s)\n", node_type,
945d2e
+                   crm_element_value(node, XML_ATTR_UNAME),
945d2e
+                   crm_element_value(node, XML_ATTR_ID));
945d2e
         }
945d2e
+        found++;
945d2e
     }
945d2e
+    // @TODO List Pacemaker Remote nodes that don't have a <node> entry
945d2e
 
945d2e
     if (found == 0) {
945d2e
-        printf("NO nodes configured\n");
945d2e
+        printf("No nodes configured\n");
945d2e
     }
945d2e
-
945d2e
-    return found;
945d2e
 }
945d2e
-- 
945d2e
1.8.3.1
945d2e