From e34421b2608f235c6a77eb6de4596d93a2128be1 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Fri, 3 Apr 2020 11:13:52 -0500
Subject: [PATCH 1/6] Refactor: libcrmcommon: new model for daemon IPC API
This is based on tools/crm_resource_controller.[ch] and the node removal code
in tools/crm_node.c, but generalized for any daemon. As of this commit, no
specific daemon is supported.
---
include/crm/common/internal.h | 8 +
include/crm/common/ipc.h | 95 +++++-
lib/common/crmcommon_private.h | 86 +++++
lib/common/ipc_client.c | 703 ++++++++++++++++++++++++++++++++++++++++-
lib/common/mainloop.c | 70 ++--
5 files changed, 940 insertions(+), 22 deletions(-)
diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h
index 28b20b4..13c29ea 100644
--- a/include/crm/common/internal.h
+++ b/include/crm/common/internal.h
@@ -19,6 +19,7 @@
#include <libxml/tree.h> // xmlNode
#include <crm/common/util.h> // crm_strdup_printf()
+#include <crm/common/mainloop.h> // mainloop_io_t, struct ipc_client_callbacks
// Internal ACL-related utilities (from acl.c)
@@ -103,6 +104,13 @@ pcmk__open_devnull(int flags)
} while (0)
+/* internal main loop utilities (from mainloop.c) */
+
+int pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata,
+ struct ipc_client_callbacks *callbacks,
+ mainloop_io_t **source);
+
+
/* internal procfs utilities (from procfs.c) */
pid_t pcmk__procfs_pid_of(const char *name);
diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h
index a0df956..8dee1b1 100644
--- a/include/crm/common/ipc.h
+++ b/include/crm/common/ipc.h
@@ -16,7 +16,8 @@ extern "C" {
/**
* \file
- * \brief Wrappers for and extensions to libqb IPC
+ * \brief IPC interface to Pacemaker daemons
+ *
* \ingroup core
*/
@@ -48,6 +49,96 @@ xmlNode *create_request_adv(const char *task, xmlNode *xml_data,
const char *origin);
+/*
+ * The library supports two methods of creating IPC connections. The older code
+ * allows connecting to any arbitrary IPC name. The newer code only allows
+ * connecting to one of the Pacemaker daemons.
+ *
+ * As daemons are converted to use the new model, the old functions should be
+ * considered deprecated for use with those daemons. Once all daemons are
+ * converted, the old functions should be officially deprecated as public API
+ * and eventually made internal API.
+ */
+
+/*
+ * Pacemaker daemon IPC
+ */
+
+//! Available IPC interfaces
+enum pcmk_ipc_server {
+ pcmk_ipc_attrd, //!< Attribute manager
+ pcmk_ipc_based, //!< CIB manager
+ pcmk_ipc_controld, //!< Controller
+ pcmk_ipc_execd, //!< Executor
+ pcmk_ipc_fenced, //!< Fencer
+ pcmk_ipc_pacemakerd, //!< Launcher
+ pcmk_ipc_schedulerd, //!< Scheduler
+};
+
+//! Possible event types that an IPC event callback can be called for
+enum pcmk_ipc_event {
+ pcmk_ipc_event_connect, //!< Result of asynchronous connection attempt
+ pcmk_ipc_event_disconnect, //!< Termination of IPC connection
+ pcmk_ipc_event_reply, //!< Daemon's reply to client IPC request
+ pcmk_ipc_event_notify, //!< Notification from daemon
+};
+
+//! How IPC replies should be dispatched
+enum pcmk_ipc_dispatch {
+ pcmk_ipc_dispatch_main, //!< Attach IPC to GMainLoop for dispatch
+ pcmk_ipc_dispatch_poll, //!< Caller will poll and dispatch IPC
+ pcmk_ipc_dispatch_sync, //!< Sending a command will wait for any reply
+};
+
+//! Client connection to Pacemaker IPC
+typedef struct pcmk_ipc_api_s pcmk_ipc_api_t;
+
+/*!
+ * \brief Callback function type for Pacemaker daemon IPC APIs
+ *
+ * \param[in] api IPC API connection
+ * \param[in] event_type The type of event that occurred
+ * \param[in] status Event status
+ * \param[in] event_data Event-specific data
+ * \param[in] user_data Caller data provided when callback was registered
+ *
+ * \note For connection and disconnection events, event_data may be NULL (for
+ * local IPC) or the name of the connected node (for remote IPC, for
+ * daemons that support that). For reply and notify events, event_data is
+ * defined by the specific daemon API.
+ */
+typedef void (*pcmk_ipc_callback_t)(pcmk_ipc_api_t *api,
+ enum pcmk_ipc_event event_type,
+ crm_exit_t status,
+ void *event_data, void *user_data);
+
+int pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server);
+
+void pcmk_free_ipc_api(pcmk_ipc_api_t *api);
+
+int pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type);
+
+void pcmk_disconnect_ipc(pcmk_ipc_api_t *api);
+
+int pcmk_poll_ipc(pcmk_ipc_api_t *api, int timeout_ms);
+
+void pcmk_dispatch_ipc(pcmk_ipc_api_t *api);
+
+void pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb,
+ void *user_data);
+
+const char *pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log);
+
+bool pcmk_ipc_is_connected(pcmk_ipc_api_t *api);
+
+int pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name,
+ uint32_t nodeid);
+
+
+/*
+ * Generic IPC API (to eventually be deprecated as public API and made internal)
+ */
+
/* *INDENT-OFF* */
enum crm_ipc_flags
{
@@ -58,7 +149,7 @@ enum crm_ipc_flags
crm_ipc_proxied = 0x00000100, /* _ALL_ replies to proxied connections need to be sent as events */
crm_ipc_client_response = 0x00000200, /* A Response is expected in reply */
- // These are options only for pcmk__ipc_send_iov()
+ // These are options for Pacemaker's internal use only (pcmk__ipc_send_*())
crm_ipc_server_event = 0x00010000, /* Send an Event instead of a Response */
crm_ipc_server_free = 0x00020000, /* Free the iovec after sending */
crm_ipc_proxied_relay_response = 0x00040000, /* all replies to proxied connections are sent as events, this flag preserves whether the event should be treated as an actual event, or a response.*/
diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h
index d06fa20..f9df27d 100644
--- a/lib/common/crmcommon_private.h
+++ b/lib/common/crmcommon_private.h
@@ -103,6 +103,84 @@ pcmk__xml_attr_value(const xmlAttr *attr)
#define PCMK__IPC_VERSION 1
+// IPC behavior that varies by daemon
+typedef struct pcmk__ipc_methods_s {
+ /*!
+ * \internal
+ * \brief Allocate any private data needed by daemon IPC
+ *
+ * \param[in] api IPC API connection
+ *
+ * \return Standard Pacemaker return code
+ */
+ int (*new_data)(pcmk_ipc_api_t *api);
+
+ /*!
+ * \internal
+ * \brief Free any private data used by daemon IPC
+ *
+ * \param[in] api_data Data allocated by new_data() method
+ */
+ void (*free_data)(void *api_data);
+
+ /*!
+ * \internal
+ * \brief Perform daemon-specific handling after successful connection
+ *
+ * Some daemons require clients to register before sending any other
+ * commands. The controller requires a CRM_OP_HELLO (with no reply), and
+ * the CIB manager, executor, and fencer require a CRM_OP_REGISTER (with a
+ * reply). Ideally this would be consistent across all daemons, but for now
+ * this allows each to do its own authorization.
+ *
+ * \param[in] api IPC API connection
+ *
+ * \return Standard Pacemaker return code
+ */
+ int (*post_connect)(pcmk_ipc_api_t *api);
+
+ /*!
+ * \internal
+ * \brief Check whether an IPC request results in a reply
+ *
+ * \parma[in] api IPC API connection
+ * \param[in] request IPC request XML
+ *
+ * \return true if request would result in an IPC reply, false otherwise
+ */
+ bool (*reply_expected)(pcmk_ipc_api_t *api, xmlNode *request);
+
+ /*!
+ * \internal
+ * \brief Perform daemon-specific handling of an IPC message
+ *
+ * \param[in] api IPC API connection
+ * \param[in] msg Message read from IPC connection
+ */
+ void (*dispatch)(pcmk_ipc_api_t *api, xmlNode *msg);
+
+ /*!
+ * \internal
+ * \brief Perform daemon-specific handling of an IPC disconnect
+ *
+ * \param[in] api IPC API connection
+ */
+ void (*post_disconnect)(pcmk_ipc_api_t *api);
+} pcmk__ipc_methods_t;
+
+// Implementation of pcmk_ipc_api_t
+struct pcmk_ipc_api_s {
+ enum pcmk_ipc_server server; // Daemon this IPC API instance is for
+ enum pcmk_ipc_dispatch dispatch_type; // How replies should be dispatched
+ crm_ipc_t *ipc; // IPC connection
+ mainloop_io_t *mainloop_io; // If using mainloop, I/O source for IPC
+ bool free_on_disconnect; // Whether disconnect should free object
+ pcmk_ipc_callback_t cb; // Caller-registered callback (if any)
+ void *user_data; // Caller-registered data (if any)
+ void *api_data; // For daemon-specific use
+ pcmk__ipc_methods_t *cmds; // Behavior that varies by daemon
+};
+
typedef struct pcmk__ipc_header_s {
struct qb_ipc_response_header qb;
uint32_t size_uncompressed;
@@ -112,6 +190,14 @@ typedef struct pcmk__ipc_header_s {
} pcmk__ipc_header_t;
G_GNUC_INTERNAL
+int pcmk__send_ipc_request(pcmk_ipc_api_t *api, xmlNode *request);
+
+G_GNUC_INTERNAL
+void pcmk__call_ipc_callback(pcmk_ipc_api_t *api,
+ enum pcmk_ipc_event event_type,
+ crm_exit_t status, void *event_data);
+
+G_GNUC_INTERNAL
unsigned int pcmk__ipc_buffer_size(unsigned int max);
G_GNUC_INTERNAL
diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c
index 7737588..16dc9b5 100644
--- a/lib/common/ipc_client.c
+++ b/lib/common/ipc_client.c
@@ -31,6 +31,679 @@
#include <crm/common/ipc_internal.h>
#include "crmcommon_private.h"
+/*!
+ * \brief Create a new object for using Pacemaker daemon IPC
+ *
+ * \param[out] api Where to store new IPC object
+ * \param[in] server Which Pacemaker daemon the object is for
+ *
+ * \return Standard Pacemaker result code
+ *
+ * \note The caller is responsible for freeing *api using pcmk_free_ipc_api().
+ * \note This is intended to supersede crm_ipc_new() but is not yet usable.
+ */
+int
+pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
+{
+ size_t max_size = 0;
+
+ if (api == NULL) {
+ return EINVAL;
+ }
+
+ *api = calloc(1, sizeof(pcmk_ipc_api_t));
+ if (*api == NULL) {
+ return errno;
+ }
+
+ (*api)->server = server;
+ if (pcmk_ipc_name(*api, false) == NULL) {
+ pcmk_free_ipc_api(*api);
+ *api = NULL;
+ return EOPNOTSUPP;
+ }
+
+ // Set server methods and max_size (if not default)
+ switch (server) {
+ case pcmk_ipc_attrd:
+ break;
+
+ case pcmk_ipc_based:
+ max_size = 512 * 1024; // 512KB
+ break;
+
+ case pcmk_ipc_controld:
+ break;
+
+ case pcmk_ipc_execd:
+ break;
+
+ case pcmk_ipc_fenced:
+ break;
+
+ case pcmk_ipc_pacemakerd:
+ break;
+
+ case pcmk_ipc_schedulerd:
+ // @TODO max_size could vary by client, maybe take as argument?
+ max_size = 5 * 1024 * 1024; // 5MB
+ break;
+ }
+ if ((*api)->cmds == NULL) {
+ pcmk_free_ipc_api(*api);
+ *api = NULL;
+ return ENOMEM;
+ }
+
+ (*api)->ipc = crm_ipc_new(pcmk_ipc_name(*api, false), max_size);
+ if ((*api)->ipc == NULL) {
+ pcmk_free_ipc_api(*api);
+ *api = NULL;
+ return ENOMEM;
+ }
+
+ // If daemon API has its own data to track, allocate it
+ if ((*api)->cmds->new_data != NULL) {
+ if ((*api)->cmds->new_data(*api) != pcmk_rc_ok) {
+ pcmk_free_ipc_api(*api);
+ *api = NULL;
+ return ENOMEM;
+ }
+ }
+ crm_trace("Created %s API IPC object", pcmk_ipc_name(*api, true));
+ return pcmk_rc_ok;
+}
+
+static void
+free_daemon_specific_data(pcmk_ipc_api_t *api)
+{
+ if ((api != NULL) && (api->cmds != NULL)) {
+ if ((api->cmds->free_data != NULL) && (api->api_data != NULL)) {
+ api->cmds->free_data(api->api_data);
+ api->api_data = NULL;
+ }
+ free(api->cmds);
+ api->cmds = NULL;
+ }
+}
+
+/*!
+ * \internal
+ * \brief Call an IPC API event callback, if one is registed
+ *
+ * \param[in] api IPC API connection
+ * \param[in] event_type The type of event that occurred
+ * \param[in] status Event status
+ * \param[in] event_data Event-specific data
+ */
+void
+pcmk__call_ipc_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
+ crm_exit_t status, void *event_data)
+{
+ if ((api != NULL) && (api->cb != NULL)) {
+ api->cb(api, event_type, status, event_data, api->user_data);
+ }
+}
+
+/*!
+ * \internal
+ * \brief Clean up after an IPC disconnect
+ *
+ * \param[in] user_data IPC API connection that disconnected
+ *
+ * \note This function can be used as a main loop IPC destroy callback.
+ */
+static void
+ipc_post_disconnect(gpointer user_data)
+{
+ pcmk_ipc_api_t *api = user_data;
+
+ crm_info("Disconnected from %s IPC API", pcmk_ipc_name(api, true));
+
+ // Perform any daemon-specific handling needed
+ if ((api->cmds != NULL) && (api->cmds->post_disconnect != NULL)) {
+ api->cmds->post_disconnect(api);
+ }
+
+ // Call client's registered event callback
+ pcmk__call_ipc_callback(api, pcmk_ipc_event_disconnect, CRM_EX_DISCONNECT,
+ NULL);
+
+ /* If this is being called from a running main loop, mainloop_gio_destroy()
+ * will free ipc and mainloop_io immediately after calling this function.
+ * If this is called from a stopped main loop, these will leak, so the best
+ * practice is to close the connection before stopping the main loop.
+ */
+ api->ipc = NULL;
+ api->mainloop_io = NULL;
+
+ if (api->free_on_disconnect) {
+ /* pcmk_free_ipc_api() has already been called, but did not free api
+ * or api->cmds because this function needed them. Do that now.
+ */
+ free_daemon_specific_data(api);
+ crm_trace("Freeing IPC API object after disconnect");
+ free(api);
+ }
+}
+
+/*!
+ * \brief Free the contents of an IPC API object
+ *
+ * \param[in] api IPC API object to free
+ */
+void
+pcmk_free_ipc_api(pcmk_ipc_api_t *api)
+{
+ bool free_on_disconnect = false;
+
+ if (api == NULL) {
+ return;
+ }
+ crm_debug("Releasing %s IPC API", pcmk_ipc_name(api, true));
+
+ if (api->ipc != NULL) {
+ if (api->mainloop_io != NULL) {
+ /* We need to keep the api pointer itself around, because it is the
+ * user data for the IPC client destroy callback. That will be
+ * triggered by the pcmk_disconnect_ipc() call below, but it might
+ * happen later in the main loop (if still running).
+ *
+ * This flag tells the destroy callback to free the object. It can't
+ * do that unconditionally, because the application might call this
+ * function after a disconnect that happened by other means.
+ */
+ free_on_disconnect = api->free_on_disconnect = true;
+ }
+ pcmk_disconnect_ipc(api); // Frees api if free_on_disconnect is true
+ }
+ if (!free_on_disconnect) {
+ free_daemon_specific_data(api);
+ crm_trace("Freeing IPC API object");
+ free(api);
+ }
+}
+
+/*!
+ * \brief Get the IPC name used with an IPC API connection
+ *
+ * \param[in] api IPC API connection
+ * \param[in] for_log If true, return human-friendly name instead of IPC name
+ *
+ * \return IPC API's human-friendly or connection name, or if none is available,
+ * "Pacemaker" if for_log is true and NULL if for_log is false
+ */
+const char *
+pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log)
+{
+ if (api == NULL) {
+ return for_log? "Pacemaker" : NULL;
+ }
+ switch (api->server) {
+ case pcmk_ipc_attrd:
+ return for_log? "attribute manager" : NULL /* T_ATTRD */;
+
+ case pcmk_ipc_based:
+ return for_log? "CIB manager" : NULL /* PCMK__SERVER_BASED_RW */;
+
+ case pcmk_ipc_controld:
+ return for_log? "controller" : NULL /* CRM_SYSTEM_CRMD */;
+
+ case pcmk_ipc_execd:
+ return for_log? "executor" : NULL /* CRM_SYSTEM_LRMD */;
+
+ case pcmk_ipc_fenced:
+ return for_log? "fencer" : NULL /* "stonith-ng" */;
+
+ case pcmk_ipc_pacemakerd:
+ return for_log? "launcher" : NULL /* CRM_SYSTEM_MCP */;
+
+ case pcmk_ipc_schedulerd:
+ return for_log? "scheduler" : NULL /* CRM_SYSTEM_PENGINE */;
+
+ default:
+ return for_log? "Pacemaker" : NULL;
+ }
+}
+
+/*!
+ * \brief Check whether an IPC API connection is active
+ *
+ * \param[in] api IPC API connection
+ *
+ * \return true if IPC is connected, false otherwise
+ */
+bool
+pcmk_ipc_is_connected(pcmk_ipc_api_t *api)
+{
+ return (api != NULL) && crm_ipc_connected(api->ipc);
+}
+
+/*!
+ * \internal
+ * \brief Call the daemon-specific API's dispatch function
+ *
+ * Perform daemon-specific handling of IPC reply dispatch. It is the daemon
+ * method's responsibility to call the client's registered event callback, as
+ * well as allocate and free any event data.
+ *
+ * \param[in] api IPC API connection
+ */
+static void
+call_api_dispatch(pcmk_ipc_api_t *api, xmlNode *message)
+{
+ crm_log_xml_trace(message, "ipc-received");
+ if ((api->cmds != NULL) && (api->cmds->dispatch != NULL)) {
+ api->cmds->dispatch(api, message);
+ }
+}
+
+/*!
+ * \internal
+ * \brief Dispatch data read from IPC source
+ *
+ * \param[in] buffer Data read from IPC
+ * \param[in] length Number of bytes of data in buffer (ignored)
+ * \param[in] user_data IPC object
+ *
+ * \return Always 0 (meaning connection is still required)
+ *
+ * \note This function can be used as a main loop IPC dispatch callback.
+ */
+static int
+dispatch_ipc_data(const char *buffer, ssize_t length, gpointer user_data)
+{
+ pcmk_ipc_api_t *api = user_data;
+ xmlNode *msg;
+
+ CRM_CHECK(api != NULL, return 0);
+
+ if (buffer == NULL) {
+ crm_warn("Empty message received from %s IPC",
+ pcmk_ipc_name(api, true));
+ return 0;
+ }
+
+ msg = string2xml(buffer);
+ if (msg == NULL) {
+ crm_warn("Malformed message received from %s IPC",
+ pcmk_ipc_name(api, true));
+ return 0;
+ }
+ call_api_dispatch(api, msg);
+ free_xml(msg);
+ return 0;
+}
+
+/*!
+ * \brief Check whether an IPC connection has data available (without main loop)
+ *
+ * \param[in] api IPC API connection
+ * \param[in] timeout_ms If less than 0, poll indefinitely; if 0, poll once
+ * and return immediately; otherwise, poll for up to
+ * this many milliseconds
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
+ * this function to check whether IPC data is available. Return values of
+ * interest include pcmk_rc_ok meaning data is available, and EAGAIN
+ * meaning no data is available; all other values indicate errors.
+ * \todo This does not allow the caller to poll multiple file descriptors at
+ * once. If there is demand for that, we could add a wrapper for
+ * crm_ipc_get_fd(api->ipc), so the caller can call poll() themselves.
+ */
+int
+pcmk_poll_ipc(pcmk_ipc_api_t *api, int timeout_ms)
+{
+ int rc;
+ struct pollfd pollfd = { 0, };
+
+ if ((api == NULL) || (api->dispatch_type != pcmk_ipc_dispatch_poll)) {
+ return EINVAL;
+ }
+ pollfd.fd = crm_ipc_get_fd(api->ipc);
+ pollfd.events = POLLIN;
+ rc = poll(&pollfd, 1, timeout_ms);
+ if (rc < 0) {
+ return errno;
+ } else if (rc == 0) {
+ return EAGAIN;
+ }
+ return pcmk_rc_ok;
+}
+
+/*!
+ * \brief Dispatch available messages on an IPC connection (without main loop)
+ *
+ * \param[in] api IPC API connection
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
+ * this function when IPC data is available.
+ */
+void
+pcmk_dispatch_ipc(pcmk_ipc_api_t *api)
+{
+ if (api == NULL) {
+ return;
+ }
+ while (crm_ipc_ready(api->ipc)) {
+ if (crm_ipc_read(api->ipc) > 0) {
+ dispatch_ipc_data(crm_ipc_buffer(api->ipc), 0, api);
+ }
+ }
+}
+
+// \return Standard Pacemaker return code
+static int
+connect_with_main_loop(pcmk_ipc_api_t *api)
+{
+ int rc;
+
+ struct ipc_client_callbacks callbacks = {
+ .dispatch = dispatch_ipc_data,
+ .destroy = ipc_post_disconnect,
+ };
+
+ rc = pcmk__add_mainloop_ipc(api->ipc, G_PRIORITY_DEFAULT, api,
+ &callbacks, &(api->mainloop_io));
+ if (rc != pcmk_rc_ok) {
+ return rc;
+ }
+ crm_debug("Connected to %s IPC (attached to main loop)",
+ pcmk_ipc_name(api, true));
+ /* After this point, api->mainloop_io owns api->ipc, so api->ipc
+ * should not be explicitly freed.
+ */
+ return pcmk_rc_ok;
+}
+
+// \return Standard Pacemaker return code
+static int
+connect_without_main_loop(pcmk_ipc_api_t *api)
+{
+ int rc;
+
+ if (!crm_ipc_connect(api->ipc)) {
+ rc = errno;
+ crm_ipc_close(api->ipc);
+ return rc;
+ }
+ crm_debug("Connected to %s IPC (without main loop)",
+ pcmk_ipc_name(api, true));
+ return pcmk_rc_ok;
+}
+
+/*!
+ * \brief Connect to a Pacemaker daemon via IPC
+ *
+ * \param[in] api IPC API instance
+ * \param[out] dispatch_type How IPC replies should be dispatched
+ *
+ * \return Standard Pacemaker return code
+ */
+int
+pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type)
+{
+ int rc = pcmk_rc_ok;
+
+ if ((api == NULL) || (api->ipc == NULL)) {
+ crm_err("Cannot connect to uninitialized API object");
+ return EINVAL;
+ }
+
+ if (crm_ipc_connected(api->ipc)) {
+ crm_trace("Already connected to %s IPC API", pcmk_ipc_name(api, true));
+ return pcmk_rc_ok;
+ }
+
+ api->dispatch_type = dispatch_type;
+ switch (dispatch_type) {
+ case pcmk_ipc_dispatch_main:
+ rc = connect_with_main_loop(api);
+ break;
+
+ case pcmk_ipc_dispatch_sync:
+ case pcmk_ipc_dispatch_poll:
+ rc = connect_without_main_loop(api);
+ break;
+ }
+ if (rc != pcmk_rc_ok) {
+ return rc;
+ }
+
+ if ((api->cmds != NULL) && (api->cmds->post_connect != NULL)) {
+ rc = api->cmds->post_connect(api);
+ if (rc != pcmk_rc_ok) {
+ crm_ipc_close(api->ipc);
+ }
+ }
+ return rc;
+}
+
+/*!
+ * \brief Disconnect an IPC API instance
+ *
+ * \param[in] api IPC API connection
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note If the connection is attached to a main loop, this function should be
+ * called before quitting the main loop, to ensure that all memory is
+ * freed.
+ */
+void
+pcmk_disconnect_ipc(pcmk_ipc_api_t *api)
+{
+ if ((api == NULL) || (api->ipc == NULL)) {
+ return;
+ }
+ switch (api->dispatch_type) {
+ case pcmk_ipc_dispatch_main:
+ {
+ mainloop_io_t *mainloop_io = api->mainloop_io;
+
+ // Make sure no code with access to api can use these again
+ api->mainloop_io = NULL;
+ api->ipc = NULL;
+
+ mainloop_del_ipc_client(mainloop_io);
+ // After this point api might have already been freed
+ }
+ break;
+
+ case pcmk_ipc_dispatch_poll:
+ case pcmk_ipc_dispatch_sync:
+ {
+ crm_ipc_t *ipc = api->ipc;
+
+ // Make sure no code with access to api can use ipc again
+ api->ipc = NULL;
+
+ // This should always be the case already, but to be safe
+ api->free_on_disconnect = false;
+
+ crm_ipc_destroy(ipc);
+ ipc_post_disconnect(api);
+ }
+ break;
+ }
+}
+
+/*!
+ * \brief Register a callback for IPC API events
+ *
+ * \param[in] api IPC API connection
+ * \param[in] callback Callback to register
+ * \param[in] userdata Caller data to pass to callback
+ *
+ * \note This function may be called multiple times to update the callback
+ * and/or user data. The caller remains responsible for freeing
+ * userdata in any case (after the IPC is disconnected, if the
+ * user data is still registered with the IPC).
+ */
+void
+pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb,
+ void *user_data)
+{
+ if (api == NULL) {
+ return;
+ }
+ api->cb = cb;
+ api->user_data = user_data;
+}
+
+/*!
+ * \internal
+ * \brief Send an XML request across an IPC API connection
+ *
+ * \param[in] api IPC API connection
+ * \param[in] request XML request to send
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note Daemon-specific IPC API functions should call this function to send
+ * requests, because it handles different dispatch types appropriately.
+ */
+int
+pcmk__send_ipc_request(pcmk_ipc_api_t *api, xmlNode *request)
+{
+ int rc;
+ xmlNode *reply = NULL;
+ enum crm_ipc_flags flags = crm_ipc_flags_none;
+
+ if ((api == NULL) || (api->ipc == NULL) || (request == NULL)) {
+ return EINVAL;
+ }
+ crm_log_xml_trace(request, "ipc-sent");
+
+ // Synchronous dispatch requires waiting for a reply
+ if ((api->dispatch_type == pcmk_ipc_dispatch_sync)
+ && (api->cmds != NULL)
+ && (api->cmds->reply_expected != NULL)
+ && (api->cmds->reply_expected(api, request))) {
+ flags = crm_ipc_client_response;
+ }
+
+ // The 0 here means a default timeout of 5 seconds
+ rc = crm_ipc_send(api->ipc, request, flags, 0, &reply);
+
+ if (rc < 0) {
+ return pcmk_legacy2rc(rc);
+ } else if (rc == 0) {
+ return ENODATA;
+ }
+
+ // With synchronous dispatch, we dispatch any reply now
+ if (reply != NULL) {
+ call_api_dispatch(api, reply);
+ free_xml(reply);
+ }
+ return pcmk_rc_ok;
+}
+
+/*!
+ * \internal
+ * \brief Create the XML for an IPC request to purge a node from the peer cache
+ *
+ * \param[in] api IPC API connection
+ * \param[in] node_name If not NULL, name of node to purge
+ * \param[in] nodeid If not 0, node ID of node to purge
+ *
+ * \return Newly allocated IPC request XML
+ *
+ * \note The controller, fencer, and pacemakerd use the same request syntax, but
+ * the attribute manager uses a different one. The CIB manager doesn't
+ * have any syntax for it. The executor and scheduler don't connect to the
+ * cluster layer and thus don't have or need any syntax for it.
+ *
+ * \todo Modify the attribute manager to accept the common syntax (as well
+ * as its current one, for compatibility with older clients). Modify
+ * the CIB manager to accept and honor the common syntax. Modify the
+ * executor and scheduler to accept the syntax (immediately returning
+ * success), just for consistency. Modify this function to use the
+ * common syntax with all daemons if their version supports it.
+ */
+static xmlNode *
+create_purge_node_request(pcmk_ipc_api_t *api, const char *node_name,
+ uint32_t nodeid)
+{
+ xmlNode *request = NULL;
+ const char *client = crm_system_name? crm_system_name : "client";
+
+ switch (api->server) {
+ case pcmk_ipc_attrd:
+ request = create_xml_node(NULL, __FUNCTION__);
+ crm_xml_add(request, F_TYPE, T_ATTRD);
+ crm_xml_add(request, F_ORIG, crm_system_name);
+ crm_xml_add(request, PCMK__XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE);
+ crm_xml_add(request, PCMK__XA_ATTR_NODE_NAME, node_name);
+ if (nodeid > 0) {
+ crm_xml_add_int(request, PCMK__XA_ATTR_NODE_ID, (int) nodeid);
+ }
+ break;
+
+ case pcmk_ipc_controld:
+ case pcmk_ipc_fenced:
+ case pcmk_ipc_pacemakerd:
+ request = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL,
+ pcmk_ipc_name(api, false), client, NULL);
+ if (nodeid > 0) {
+ crm_xml_set_id(request, "%lu", (unsigned long) nodeid);
+ }
+ crm_xml_add(request, XML_ATTR_UNAME, node_name);
+ break;
+
+ case pcmk_ipc_based:
+ case pcmk_ipc_execd:
+ case pcmk_ipc_schedulerd:
+ break;
+ }
+ return request;
+}
+
+/*!
+ * \brief Ask a Pacemaker daemon to purge a node from its peer cache
+ *
+ * \param[in] api IPC API connection
+ * \param[in] node_name If not NULL, name of node to purge
+ * \param[in] nodeid If not 0, node ID of node to purge
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note At least one of node_name or nodeid must be specified.
+ */
+int
+pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name, uint32_t nodeid)
+{
+ int rc = 0;
+ xmlNode *request = NULL;
+
+ if (api == NULL) {
+ return EINVAL;
+ }
+ if ((node_name == NULL) && (nodeid == 0)) {
+ return EINVAL;
+ }
+
+ request = create_purge_node_request(api, node_name, nodeid);
+ if (request == NULL) {
+ return EOPNOTSUPP;
+ }
+ rc = pcmk__send_ipc_request(api, request);
+ free_xml(request);
+
+ crm_debug("%s peer cache purge of node %s[%lu]: rc=%d",
+ pcmk_ipc_name(api, true), node_name, (unsigned long) nodeid, rc);
+ return rc;
+}
+
+/*
+ * Generic IPC API (to eventually be deprecated as public API and made internal)
+ */
+
struct crm_ipc_s {
struct pollfd pfd;
unsigned int max_buf_size; // maximum bytes we can send or receive over IPC
@@ -42,16 +715,44 @@ struct crm_ipc_s {
qb_ipcc_connection_t *ipc;
};
+/*!
+ * \brief Create a new (legacy) object for using Pacemaker daemon IPC
+ *
+ * \param[in] name IPC system name to connect to
+ * \param[in] max_size Use a maximum IPC buffer size of at least this size
+ *
+ * \return Newly allocated IPC object on success, NULL otherwise
+ *
+ * \note The caller is responsible for freeing the result using
+ * crm_ipc_destroy().
+ * \note This should be considered deprecated for use with daemons supported by
+ * pcmk_new_ipc_api().
+ */
crm_ipc_t *
crm_ipc_new(const char *name, size_t max_size)
{
crm_ipc_t *client = NULL;
client = calloc(1, sizeof(crm_ipc_t));
+ if (client == NULL) {
+ crm_err("Could not create IPC connection: %s", strerror(errno));
+ return NULL;
+ }
client->name = strdup(name);
+ if (client->name == NULL) {
+ crm_err("Could not create IPC connection: %s", strerror(errno));
+ free(client);
+ return NULL;
+ }
client->buf_size = pcmk__ipc_buffer_size(max_size);
client->buffer = malloc(client->buf_size);
+ if (client->buffer == NULL) {
+ crm_err("Could not create IPC connection: %s", strerror(errno));
+ free(client->name);
+ free(client);
+ return NULL;
+ }
/* Clients initiating connection pick the max buf size */
client->max_buf_size = client->buf_size;
@@ -143,8 +844,6 @@ void
crm_ipc_close(crm_ipc_t * client)
{
if (client) {
- crm_trace("Disconnecting %s IPC connection %p (%p)", client->name, client, client->ipc);
-
if (client->ipc) {
qb_ipcc_connection_t *ipc = client->ipc;
diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c
index 634eead..e942e57 100644
--- a/lib/common/mainloop.c
+++ b/lib/common/mainloop.c
@@ -834,32 +834,66 @@ mainloop_gio_destroy(gpointer c)
free(c_name);
}
-mainloop_io_t *
-mainloop_add_ipc_client(const char *name, int priority, size_t max_size, void *userdata,
- struct ipc_client_callbacks *callbacks)
+/*!
+ * \brief Connect to IPC and add it as a main loop source
+ *
+ * \param[in] ipc IPC connection to add
+ * \param[in] priority Event source priority to use for connection
+ * \param[in] userdata Data to register with callbacks
+ * \param[in] callbacks Dispatch and destroy callbacks for connection
+ * \param[out] source Newly allocated event source
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note On failure, the caller is still responsible for ipc. On success, the
+ * caller should call mainloop_del_ipc_client() when source is no longer
+ * needed, which will lead to the disconnection of the IPC later in the
+ * main loop if it is connected. However the IPC disconnects,
+ * mainloop_gio_destroy() will free ipc and source after calling the
+ * destroy callback.
+ */
+int
+pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata,
+ struct ipc_client_callbacks *callbacks,
+ mainloop_io_t **source)
{
- mainloop_io_t *client = NULL;
- crm_ipc_t *conn = crm_ipc_new(name, max_size);
+ CRM_CHECK((ipc != NULL) && (callbacks != NULL), return EINVAL);
- if (conn && crm_ipc_connect(conn)) {
- int32_t fd = crm_ipc_get_fd(conn);
+ if (!crm_ipc_connect(ipc)) {
+ return ENOTCONN;
+ }
+ *source = mainloop_add_fd(crm_ipc_name(ipc), priority, crm_ipc_get_fd(ipc),
+ userdata, NULL);
+ if (*source == NULL) {
+ int rc = errno;
- client = mainloop_add_fd(name, priority, fd, userdata, NULL);
+ crm_ipc_close(ipc);
+ return rc;
}
+ (*source)->ipc = ipc;
+ (*source)->destroy_fn = callbacks->destroy;
+ (*source)->dispatch_fn_ipc = callbacks->dispatch;
+ return pcmk_rc_ok;
+}
- if (client == NULL) {
- crm_perror(LOG_TRACE, "Connection to %s failed", name);
- if (conn) {
- crm_ipc_close(conn);
- crm_ipc_destroy(conn);
+mainloop_io_t *
+mainloop_add_ipc_client(const char *name, int priority, size_t max_size,
+ void *userdata, struct ipc_client_callbacks *callbacks)
+{
+ crm_ipc_t *ipc = crm_ipc_new(name, max_size);
+ mainloop_io_t *source = NULL;
+ int rc = pcmk__add_mainloop_ipc(ipc, priority, userdata, callbacks,
+ &source);
+
+ if (rc != pcmk_rc_ok) {
+ if (crm_log_level == LOG_STDOUT) {
+ fprintf(stderr, "Connection to %s failed: %s",
+ name, pcmk_rc_str(rc));
}
+ crm_ipc_destroy(ipc);
return NULL;
}
-
- client->ipc = conn;
- client->destroy_fn = callbacks->destroy;
- client->dispatch_fn_ipc = callbacks->dispatch;
- return client;
+ return source;
}
void
--
1.8.3.1
From b9539da27998ff5e6c8b681f39603550a923ca33 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Mon, 6 Apr 2020 17:40:55 -0500
Subject: [PATCH 2/6] Refactor: libcrmcommon: add C API for controller IPC
Implement a C API for controller IPC using the new IPC API model.
---
include/crm/common/Makefile.am | 2 +-
include/crm/common/ipc.h | 4 +-
include/crm/common/ipc_controld.h | 99 +++++++
lib/common/Makefile.am | 1 +
lib/common/crmcommon_private.h | 6 +
lib/common/ipc_client.c | 46 +--
lib/common/ipc_controld.c | 609 ++++++++++++++++++++++++++++++++++++++
7 files changed, 723 insertions(+), 44 deletions(-)
create mode 100644 include/crm/common/ipc_controld.h
create mode 100644 lib/common/ipc_controld.c
diff --git a/include/crm/common/Makefile.am b/include/crm/common/Makefile.am
index 776e4a7..f29d105 100644
--- a/include/crm/common/Makefile.am
+++ b/include/crm/common/Makefile.am
@@ -12,7 +12,7 @@ MAINTAINERCLEANFILES = Makefile.in
headerdir=$(pkgincludedir)/crm/common
header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h results.h \
- nvpair.h acl.h
+ nvpair.h acl.h ipc_controld.h
noinst_HEADERS = internal.h alerts_internal.h \
iso8601_internal.h remote_internal.h xml_internal.h \
ipc_internal.h output.h cmdline_internal.h curses_internal.h \
diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h
index 8dee1b1..c67aaea 100644
--- a/include/crm/common/ipc.h
+++ b/include/crm/common/ipc.h
@@ -217,7 +217,9 @@ unsigned int crm_ipc_default_buffer_size(void);
int crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid,
pid_t *gotpid, uid_t *gotuid, gid_t *gotgid);
-/* Utils */
+/* This is controller-specific but is declared in this header for C API
+ * backward compatibility.
+ */
xmlNode *create_hello_message(const char *uuid, const char *client_name,
const char *major_version, const char *minor_version);
diff --git a/include/crm/common/ipc_controld.h b/include/crm/common/ipc_controld.h
new file mode 100644
index 0000000..0ebabfc
--- /dev/null
+++ b/include/crm/common/ipc_controld.h
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2020 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef PCMK__IPC_CONTROLD__H
+# define PCMK__IPC_CONTROLD__H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \file
+ * \brief IPC commands for Pacemaker controller
+ *
+ * \ingroup core
+ */
+
+#include <stdbool.h> // bool
+#include <libxml/tree.h> // xmlNode
+#include <crm/common/ipc.h> // pcmk_ipc_api_t
+
+//! Possible types of controller replies
+enum pcmk_controld_api_reply {
+ pcmk_controld_reply_unknown,
+ pcmk_controld_reply_reprobe,
+ pcmk_controld_reply_info,
+ pcmk_controld_reply_resource,
+ pcmk_controld_reply_ping,
+};
+
+/*!
+ * Controller reply passed to event callback
+ *
+ * \note Shutdown and election calls have no reply. Reprobe calls are
+ * acknowledged but contain no data (reply_type will be the only item
+ * set). Node info and ping calls have their own reply data. Fail and
+ * refresh calls use the resource reply type and reply data.
+ * \note The pointers in the reply are only guaranteed to be meaningful for the
+ * execution of the callback; if the values are needed for later, the
+ * callback should copy them.
+ */
+typedef struct {
+ enum pcmk_controld_api_reply reply_type;
+ const char *feature_set; //!< CRM feature set advertised by controller
+ const char *host_from; //!< Name of node that sent reply
+
+ union {
+ // pcmk_controld_reply_info
+ struct {
+ bool have_quorum;
+ bool is_remote;
+ int id;
+ const char *uuid;
+ const char *uname;
+ const char *state;
+ } node_info;
+
+ // pcmk_controld_reply_resource
+ struct {
+ xmlNode *node_state; //<! Resource operation history XML
+ } resource;
+
+ // pcmk_controld_reply_ping
+ struct {
+ const char *sys_from;
+ const char *fsa_state;
+ const char *result;
+ } ping;
+ } data;
+} pcmk_controld_api_reply_t;
+
+int pcmk_controld_api_reprobe(pcmk_ipc_api_t *api, const char *target_node,
+ const char *router_node);
+int pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid);
+int pcmk_controld_api_fail(pcmk_ipc_api_t *api, const char *target_node,
+ const char *router_node, const char *rsc_id,
+ const char *rsc_long_id, const char *standard,
+ const char *provider, const char *type);
+int pcmk_controld_api_refresh(pcmk_ipc_api_t *api, const char *target_node,
+ const char *router_node, const char *rsc_id,
+ const char *rsc_long_id, const char *standard,
+ const char *provider, const char *type,
+ bool cib_only);
+int pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name);
+int pcmk_controld_api_shutdown(pcmk_ipc_api_t *api, const char *node_name);
+int pcmk_controld_api_start_election(pcmk_ipc_api_t *api);
+unsigned int pcmk_controld_api_replies_expected(pcmk_ipc_api_t *api);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // PCMK__IPC_CONTROLD__H
diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am
index 29404a6..db66a6e 100644
--- a/lib/common/Makefile.am
+++ b/lib/common/Makefile.am
@@ -49,6 +49,7 @@ libcrmcommon_la_SOURCES += digest.c
libcrmcommon_la_SOURCES += io.c
libcrmcommon_la_SOURCES += ipc_client.c
libcrmcommon_la_SOURCES += ipc_common.c
+libcrmcommon_la_SOURCES += ipc_controld.c
libcrmcommon_la_SOURCES += ipc_server.c
libcrmcommon_la_SOURCES += iso8601.c
libcrmcommon_la_SOURCES += logging.c
diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h
index f9df27d..49dae6c 100644
--- a/lib/common/crmcommon_private.h
+++ b/lib/common/crmcommon_private.h
@@ -103,6 +103,9 @@ pcmk__xml_attr_value(const xmlAttr *attr)
#define PCMK__IPC_VERSION 1
+#define PCMK__CONTROLD_API_MAJOR "1"
+#define PCMK__CONTROLD_API_MINOR "0"
+
// IPC behavior that varies by daemon
typedef struct pcmk__ipc_methods_s {
/*!
@@ -203,4 +206,7 @@ unsigned int pcmk__ipc_buffer_size(unsigned int max);
G_GNUC_INTERNAL
bool pcmk__valid_ipc_header(const pcmk__ipc_header_t *header);
+G_GNUC_INTERNAL
+pcmk__ipc_methods_t *pcmk__controld_api_methods(void);
+
#endif // CRMCOMMON_PRIVATE__H
diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c
index 16dc9b5..4077d61 100644
--- a/lib/common/ipc_client.c
+++ b/lib/common/ipc_client.c
@@ -40,7 +40,8 @@
* \return Standard Pacemaker result code
*
* \note The caller is responsible for freeing *api using pcmk_free_ipc_api().
- * \note This is intended to supersede crm_ipc_new() but is not yet usable.
+ * \note This is intended to supersede crm_ipc_new() but currently only
+ * supports the controller IPC API.
*/
int
pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
@@ -73,6 +74,7 @@ pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
break;
case pcmk_ipc_controld:
+ (*api)->cmds = pcmk__controld_api_methods();
break;
case pcmk_ipc_execd:
@@ -247,7 +249,7 @@ pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log)
return for_log? "CIB manager" : NULL /* PCMK__SERVER_BASED_RW */;
case pcmk_ipc_controld:
- return for_log? "controller" : NULL /* CRM_SYSTEM_CRMD */;
+ return for_log? "controller" : CRM_SYSTEM_CRMD;
case pcmk_ipc_execd:
return for_log? "executor" : NULL /* CRM_SYSTEM_LRMD */;
@@ -1412,43 +1414,3 @@ bail:
}
return rc;
}
-
-xmlNode *
-create_hello_message(const char *uuid,
- const char *client_name, const char *major_version, const char *minor_version)
-{
- xmlNode *hello_node = NULL;
- xmlNode *hello = NULL;
-
- if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name)
- || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) {
- crm_err("Could not create IPC hello message from %s (UUID %s): "
- "missing information",
- client_name? client_name : "unknown client",
- uuid? uuid : "unknown");
- return NULL;
- }
-
- hello_node = create_xml_node(NULL, XML_TAG_OPTIONS);
- if (hello_node == NULL) {
- crm_err("Could not create IPC hello message from %s (UUID %s): "
- "Message data creation failed", client_name, uuid);
- return NULL;
- }
-
- crm_xml_add(hello_node, "major_version", major_version);
- crm_xml_add(hello_node, "minor_version", minor_version);
- crm_xml_add(hello_node, "client_name", client_name);
- crm_xml_add(hello_node, "client_uuid", uuid);
-
- hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
- if (hello == NULL) {
- crm_err("Could not create IPC hello message from %s (UUID %s): "
- "Request creation failed", client_name, uuid);
- return NULL;
- }
- free_xml(hello_node);
-
- crm_trace("Created hello message from %s (UUID %s)", client_name, uuid);
- return hello;
-}
diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c
new file mode 100644
index 0000000..22bb733
--- /dev/null
+++ b/lib/common/ipc_controld.c
@@ -0,0 +1,609 @@
+/*
+ * Copyright 2020 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <libxml/tree.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/common/ipc_controld.h>
+#include "crmcommon_private.h"
+
+struct controld_api_private_s {
+ char *client_uuid;
+ unsigned int replies_expected;
+};
+
+// \return Standard Pacemaker return code
+static int
+new_data(pcmk_ipc_api_t *api)
+{
+ struct controld_api_private_s *private = NULL;
+
+ api->api_data = calloc(1, sizeof(struct controld_api_private_s));
+
+ if (api->api_data == NULL) {
+ return errno;
+ }
+
+ private = api->api_data;
+
+ /* This is set to the PID because that's how it was always done, but PIDs
+ * are not unique because clients can be remote. The value appears to be
+ * unused other than as part of F_CRM_SYS_FROM in IPC requests, which is
+ * only compared against the internal system names (CRM_SYSTEM_TENGINE,
+ * etc.), so it shouldn't be a problem.
+ */
+ private->client_uuid = pcmk__getpid_s();
+
+ /* @TODO Implement a call ID model similar to the CIB, executor, and fencer
+ * IPC APIs, so that requests and replies can be matched, and
+ * duplicate replies can be discarded.
+ */
+ return pcmk_rc_ok;
+}
+
+static void
+free_data(void *data)
+{
+ free(((struct controld_api_private_s *) data)->client_uuid);
+ free(data);
+}
+
+// \return Standard Pacemaker return code
+static int
+post_connect(pcmk_ipc_api_t *api)
+{
+ /* The controller currently requires clients to register via a hello
+ * request, but does not reply back.
+ */
+ struct controld_api_private_s *private = api->api_data;
+ const char *client_name = crm_system_name? crm_system_name : "client";
+ xmlNode *hello;
+ int rc;
+
+ hello = create_hello_message(private->client_uuid, client_name,
+ PCMK__CONTROLD_API_MAJOR,
+ PCMK__CONTROLD_API_MINOR);
+ rc = pcmk__send_ipc_request(api, hello);
+ free_xml(hello);
+ if (rc != pcmk_rc_ok) {
+ crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s",
+ pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc);
+ } else {
+ crm_debug("Sent IPC hello to %s", pcmk_ipc_name(api, true));
+ }
+ return rc;
+}
+
+#define xml_true(xml, field) crm_is_true(crm_element_value(xml, field))
+
+static void
+set_node_info_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
+{
+ data->reply_type = pcmk_controld_reply_info;
+ if (msg_data == NULL) {
+ return;
+ }
+ data->data.node_info.have_quorum = xml_true(msg_data, XML_ATTR_HAVE_QUORUM);
+ data->data.node_info.is_remote = xml_true(msg_data, XML_NODE_IS_REMOTE);
+ crm_element_value_int(msg_data, XML_ATTR_ID, &(data->data.node_info.id));
+ data->data.node_info.uuid = crm_element_value(msg_data, XML_ATTR_UUID);
+ data->data.node_info.uname = crm_element_value(msg_data, XML_ATTR_UNAME);
+ data->data.node_info.state = crm_element_value(msg_data, XML_NODE_IS_PEER);
+}
+
+static void
+set_ping_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
+{
+ data->reply_type = pcmk_controld_reply_ping;
+ if (msg_data == NULL) {
+ return;
+ }
+ data->data.ping.sys_from = crm_element_value(msg_data,
+ XML_PING_ATTR_SYSFROM);
+ data->data.ping.fsa_state = crm_element_value(msg_data,
+ XML_PING_ATTR_CRMDSTATE);
+ data->data.ping.result = crm_element_value(msg_data, XML_PING_ATTR_STATUS);
+}
+
+static bool
+reply_expected(pcmk_ipc_api_t *api, xmlNode *request)
+{
+ const char *command = crm_element_value(request, F_CRM_TASK);
+
+ if (command == NULL) {
+ return false;
+ }
+
+ // We only need to handle commands that functions in this file can send
+ return !strcmp(command, CRM_OP_REPROBE)
+ || !strcmp(command, CRM_OP_NODE_INFO)
+ || !strcmp(command, CRM_OP_PING)
+ || !strcmp(command, CRM_OP_LRM_FAIL)
+ || !strcmp(command, CRM_OP_LRM_DELETE);
+}
+
+static void
+dispatch(pcmk_ipc_api_t *api, xmlNode *reply)
+{
+ struct controld_api_private_s *private = api->api_data;
+ crm_exit_t status = CRM_EX_OK;
+ xmlNode *msg_data = NULL;
+ const char *value = NULL;
+ pcmk_controld_api_reply_t reply_data = {
+ pcmk_controld_reply_unknown, NULL, NULL,
+ };
+
+ if (private->replies_expected > 0) {
+ private->replies_expected--;
+ }
+
+ // Do some basic validation of the reply
+
+ /* @TODO We should be able to verify that value is always a response, but
+ * currently the controller doesn't always properly set the type. Even
+ * if we fix the controller, we'll still need to handle replies from
+ * old versions (feature set could be used to differentiate).
+ */
+ value = crm_element_value(reply, F_CRM_MSG_TYPE);
+ if ((value == NULL) || (strcmp(value, XML_ATTR_REQUEST)
+ && strcmp(value, XML_ATTR_RESPONSE))) {
+ crm_debug("Unrecognizable controller message: invalid message type '%s'",
+ crm_str(value));
+ status = CRM_EX_PROTOCOL;
+ reply = NULL;
+ }
+
+ if (crm_element_value(reply, XML_ATTR_REFERENCE) == NULL) {
+ crm_debug("Unrecognizable controller message: no reference");
+ status = CRM_EX_PROTOCOL;
+ reply = NULL;
+ }
+
+ value = crm_element_value(reply, F_CRM_TASK);
+ if (value == NULL) {
+ crm_debug("Unrecognizable controller message: no command name");
+ status = CRM_EX_PROTOCOL;
+ reply = NULL;
+ }
+
+ // Parse useful info from reply
+
+ if (reply != NULL) {
+ reply_data.feature_set = crm_element_value(reply, XML_ATTR_VERSION);
+ reply_data.host_from = crm_element_value(reply, F_CRM_HOST_FROM);
+ msg_data = get_message_xml(reply, F_CRM_DATA);
+
+ if (!strcmp(value, CRM_OP_REPROBE)) {
+ reply_data.reply_type = pcmk_controld_reply_reprobe;
+
+ } else if (!strcmp(value, CRM_OP_NODE_INFO)) {
+ set_node_info_data(&reply_data, msg_data);
+
+ } else if (!strcmp(value, CRM_OP_INVOKE_LRM)) {
+ reply_data.reply_type = pcmk_controld_reply_resource;
+ reply_data.data.resource.node_state = msg_data;
+
+ } else if (!strcmp(value, CRM_OP_PING)) {
+ set_ping_data(&reply_data, msg_data);
+
+ } else {
+ crm_debug("Unrecognizable controller message: unknown command '%s'",
+ value);
+ status = CRM_EX_PROTOCOL;
+ reply = NULL;
+ }
+ }
+
+ pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);
+}
+
+pcmk__ipc_methods_t *
+pcmk__controld_api_methods()
+{
+ pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));
+
+ if (cmds != NULL) {
+ cmds->new_data = new_data;
+ cmds->free_data = free_data;
+ cmds->post_connect = post_connect;
+ cmds->reply_expected = reply_expected;
+ cmds->dispatch = dispatch;
+ }
+ return cmds;
+}
+
+/*!
+ * \internal
+ * \brief Create XML for a controller IPC request
+ *
+ * \param[in] api Controller connection
+ * \param[in] op Controller IPC command name
+ * \param[in] node Node name to set as destination host
+ * \param[in] msg_data XML to attach to request as message data
+ *
+ * \return Newly allocated XML for request
+ */
+static xmlNode *
+create_controller_request(pcmk_ipc_api_t *api, const char *op,
+ const char *node, xmlNode *msg_data)
+{
+ struct controld_api_private_s *private = api->api_data;
+ const char *sys_to = NULL;
+
+ if ((node == NULL) && !strcmp(op, CRM_OP_PING)) {
+ sys_to = CRM_SYSTEM_DC;
+ } else {
+ sys_to = CRM_SYSTEM_CRMD;
+ }
+ return create_request(op, msg_data, node, sys_to,
+ (crm_system_name? crm_system_name : "client"),
+ private->client_uuid);
+}
+
+// \return Standard Pacemaker return code
+static int
+send_controller_request(pcmk_ipc_api_t *api, xmlNode *request,
+ bool reply_is_expected)
+{
+ int rc;
+
+ if (crm_element_value(request, XML_ATTR_REFERENCE) == NULL) {
+ return EINVAL;
+ }
+ rc = pcmk__send_ipc_request(api, request);
+ if ((rc == pcmk_rc_ok) && reply_is_expected) {
+ struct controld_api_private_s *private = api->api_data;
+
+ private->replies_expected++;
+ }
+ return rc;
+}
+
+static xmlNode *
+create_reprobe_message_data(const char *target_node, const char *router_node)
+{
+ xmlNode *msg_data;
+
+ msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE);
+ crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
+ if ((router_node != NULL) && safe_str_neq(router_node, target_node)) {
+ crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
+ }
+ return msg_data;
+}
+
+/*!
+ * \brief Send a reprobe controller operation
+ *
+ * \param[in] api Controller connection
+ * \param[in] target_node Name of node to reprobe
+ * \param[in] router_node Router node for host
+ *
+ * \return Standard Pacemaker return code
+ * \note Event callback will get a reply of type pcmk_controld_reply_reprobe.
+ */
+int
+pcmk_controld_api_reprobe(pcmk_ipc_api_t *api, const char *target_node,
+ const char *router_node)
+{
+ xmlNode *request;
+ xmlNode *msg_data;
+ int rc = pcmk_rc_ok;
+
+ if (api == NULL) {
+ return EINVAL;
+ }
+ if (router_node == NULL) {
+ router_node = target_node;
+ }
+ crm_debug("Sending %s IPC request to reprobe %s via %s",
+ pcmk_ipc_name(api, true), crm_str(target_node),
+ crm_str(router_node));
+ msg_data = create_reprobe_message_data(target_node, router_node);
+ request = create_controller_request(api, CRM_OP_REPROBE, router_node,
+ msg_data);
+ rc = send_controller_request(api, request, true);
+ free_xml(msg_data);
+ free_xml(request);
+ return rc;
+}
+
+/*!
+ * \brief Send a "node info" controller operation
+ *
+ * \param[in] api Controller connection
+ * \param[in] nodeid ID of node to get info for (or 0 for local node)
+ *
+ * \return Standard Pacemaker return code
+ * \note Event callback will get a reply of type pcmk_controld_reply_info.
+ */
+int
+pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid)
+{
+ xmlNode *request;
+ int rc = pcmk_rc_ok;
+
+ request = create_controller_request(api, CRM_OP_NODE_INFO, NULL, NULL);
+ if (request == NULL) {
+ return EINVAL;
+ }
+ if (nodeid > 0) {
+ crm_xml_set_id(request, "%lu", (unsigned long) nodeid);
+ }
+
+ rc = send_controller_request(api, request, true);
+ free_xml(request);
+ return rc;
+}
+
+/*!
+ * \brief Ask the controller for status
+ *
+ * \param[in] api Controller connection
+ * \param[in] node_name Name of node whose status is desired (or NULL for DC)
+ *
+ * \return Standard Pacemaker return code
+ * \note Event callback will get a reply of type pcmk_controld_reply_ping.
+ */
+int
+pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name)
+{
+ xmlNode *request;
+ int rc = pcmk_rc_ok;
+
+ request = create_controller_request(api, CRM_OP_PING, node_name, NULL);
+ if (request == NULL) {
+ return EINVAL;
+ }
+ rc = send_controller_request(api, request, true);
+ free_xml(request);
+ return rc;
+}
+
+/*!
+ * \internal
+ * \brief Ask the controller to shut down
+ *
+ * \param[in] api Controller connection
+ * \param[in] node_name Name of node whose controller should shut down
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note This capability currently does not work, so the function is considered
+ * internal. It will likely be removed.
+ * \note Event callback will not get a reply.
+ */
+int
+pcmk_controld_api_shutdown(pcmk_ipc_api_t *api, const char *node_name)
+{
+ xmlNode *request;
+ int rc = pcmk_rc_ok;
+
+ request = create_controller_request(api, CRM_OP_SHUTDOWN, NULL, NULL);
+ if (request == NULL) {
+ return EINVAL;
+ }
+ rc = send_controller_request(api, request, false);
+ free_xml(request);
+ return rc;
+}
+
+/*!
+ * \internal
+ * \brief Ask the controller to start a DC election
+ *
+ * \param[in] api Controller connection
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note This capability currently does not work, so the function is considered
+ * internal. It will likely be removed.
+ * \note Event callback will not get a reply.
+ */
+int
+pcmk_controld_api_start_election(pcmk_ipc_api_t *api)
+{
+ xmlNode *request;
+ int rc = pcmk_rc_ok;
+
+ request = create_controller_request(api, CRM_OP_VOTE, NULL, NULL);
+ if (request == NULL) {
+ return EINVAL;
+ }
+ rc = send_controller_request(api, request, false);
+ free_xml(request);
+ return rc;
+}
+
+// \return Standard Pacemaker return code
+static int
+controller_resource_op(pcmk_ipc_api_t *api, const char *op,
+ const char *target_node, const char *router_node,
+ bool cib_only, const char *rsc_id,
+ const char *rsc_long_id, const char *standard,
+ const char *provider, const char *type)
+{
+ int rc = pcmk_rc_ok;
+ char *key;
+ xmlNode *request, *msg_data, *xml_rsc, *params;
+
+ if (api == NULL) {
+ return EINVAL;
+ }
+ if (router_node == NULL) {
+ router_node = target_node;
+ }
+
+ msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
+
+ /* The controller logs the transition key from resource op requests, so we
+ * need to have *something* for it.
+ * @TODO don't use "crm-resource"
+ */
+ key = pcmk__transition_key(0, getpid(), 0,
+ "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
+ crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
+ free(key);
+
+ crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
+ if (safe_str_neq(router_node, target_node)) {
+ crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
+ }
+
+ if (cib_only) {
+ // Indicate that only the CIB needs to be cleaned
+ crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB);
+ }
+
+ xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
+ crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
+ crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc_long_id);
+ crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, standard);
+ crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, provider);
+ crm_xml_add(xml_rsc, XML_ATTR_TYPE, type);
+
+ params = create_xml_node(msg_data, XML_TAG_ATTRS);
+ crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
+
+ // The controller parses the timeout from the request
+ key = crm_meta_name(XML_ATTR_TIMEOUT);
+ crm_xml_add(params, key, "60000"); /* 1 minute */ //@TODO pass as arg
+ free(key);
+
+ request = create_controller_request(api, op, router_node, msg_data);
+ rc = send_controller_request(api, request, true);
+ free_xml(msg_data);
+ free_xml(request);
+ return rc;
+}
+
+/*!
+ * \brief Ask the controller to fail a resource
+ *
+ * \param[in] api Controller connection
+ * \param[in] target_node Name of node resource is on
+ * \param[in] router_node Router node for target
+ * \param[in] rsc_id ID of resource to fail
+ * \param[in] rsc_long_id Long ID of resource (if any)
+ * \param[in] standard Standard of resource
+ * \param[in] provider Provider of resource (if any)
+ * \param[in] type Type of resource to fail
+ *
+ * \return Standard Pacemaker return code
+ * \note Event callback will get a reply of type pcmk_controld_reply_resource.
+ */
+int
+pcmk_controld_api_fail(pcmk_ipc_api_t *api,
+ const char *target_node, const char *router_node,
+ const char *rsc_id, const char *rsc_long_id,
+ const char *standard, const char *provider,
+ const char *type)
+{
+ crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s",
+ pcmk_ipc_name(api, true), crm_str(rsc_id), crm_str(rsc_long_id),
+ crm_str(target_node), crm_str(router_node));
+ return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node,
+ router_node, false, rsc_id, rsc_long_id,
+ standard, provider, type);
+}
+
+/*!
+ * \brief Ask the controller to refresh a resource
+ *
+ * \param[in] api Controller connection
+ * \param[in] target_node Name of node resource is on
+ * \param[in] router_node Router node for target
+ * \param[in] rsc_id ID of resource to refresh
+ * \param[in] rsc_long_id Long ID of resource (if any)
+ * \param[in] standard Standard of resource
+ * \param[in] provider Provider of resource (if any)
+ * \param[in] type Type of resource
+ * \param[in] cib_only If true, clean resource from CIB only
+ *
+ * \return Standard Pacemaker return code
+ * \note Event callback will get a reply of type pcmk_controld_reply_resource.
+ */
+int
+pcmk_controld_api_refresh(pcmk_ipc_api_t *api, const char *target_node,
+ const char *router_node,
+ const char *rsc_id, const char *rsc_long_id,
+ const char *standard, const char *provider,
+ const char *type, bool cib_only)
+{
+ crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s",
+ pcmk_ipc_name(api, true), crm_str(rsc_id), crm_str(rsc_long_id),
+ crm_str(target_node), crm_str(router_node));
+ return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node,
+ router_node, cib_only, rsc_id, rsc_long_id,
+ standard, provider, type);
+}
+
+/*!
+ * \brief Get the number of IPC replies currently expected from the controller
+ *
+ * \param[in] api Controller IPC API connection
+ *
+ * \return Number of replies expected
+ */
+unsigned int
+pcmk_controld_api_replies_expected(pcmk_ipc_api_t *api)
+{
+ struct controld_api_private_s *private = api->api_data;
+
+ return private->replies_expected;
+}
+
+xmlNode *
+create_hello_message(const char *uuid, const char *client_name,
+ const char *major_version, const char *minor_version)
+{
+ xmlNode *hello_node = NULL;
+ xmlNode *hello = NULL;
+
+ if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name)
+ || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) {
+ crm_err("Could not create IPC hello message from %s (UUID %s): "
+ "missing information",
+ client_name? client_name : "unknown client",
+ uuid? uuid : "unknown");
+ return NULL;
+ }
+
+ hello_node = create_xml_node(NULL, XML_TAG_OPTIONS);
+ if (hello_node == NULL) {
+ crm_err("Could not create IPC hello message from %s (UUID %s): "
+ "Message data creation failed", client_name, uuid);
+ return NULL;
+ }
+
+ crm_xml_add(hello_node, "major_version", major_version);
+ crm_xml_add(hello_node, "minor_version", minor_version);
+ crm_xml_add(hello_node, "client_name", client_name);
+ crm_xml_add(hello_node, "client_uuid", uuid);
+
+ hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
+ if (hello == NULL) {
+ crm_err("Could not create IPC hello message from %s (UUID %s): "
+ "Request creation failed", client_name, uuid);
+ return NULL;
+ }
+ free_xml(hello_node);
+
+ crm_trace("Created hello message from %s (UUID %s)", client_name, uuid);
+ return hello;
+}
--
1.8.3.1
From 1d1b34664b64f6805aeaf4d8e29e77aa8f59b4fc Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 7 Apr 2020 15:43:01 -0500
Subject: [PATCH 3/6] Refactor: tools: convert crm_resource to use new
controller IPC model
---
tools/Makefile.am | 3 +-
tools/crm_resource.c | 138 +++++++------
tools/crm_resource.h | 9 +-
tools/crm_resource_controller.c | 425 ----------------------------------------
tools/crm_resource_controller.h | 198 -------------------
tools/crm_resource_runtime.c | 30 +--
6 files changed, 96 insertions(+), 707 deletions(-)
delete mode 100644 tools/crm_resource_controller.c
delete mode 100644 tools/crm_resource_controller.h
diff --git a/tools/Makefile.am b/tools/Makefile.am
index c822a8c..4609b0f 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -12,7 +12,7 @@ if BUILD_SYSTEMD
systemdsystemunit_DATA = crm_mon.service
endif
-noinst_HEADERS = crm_mon.h crm_resource.h crm_resource_controller.h
+noinst_HEADERS = crm_mon.h crm_resource.h
pcmkdir = $(datadir)/$(PACKAGE)
pcmk_DATA = report.common report.collector
@@ -115,7 +115,6 @@ crm_attribute_LDADD = $(top_builddir)/lib/cluster/libcrmcluster.la \
crm_resource_SOURCES = crm_resource.c \
crm_resource_ban.c \
- crm_resource_controller.c \
crm_resource_print.c \
crm_resource_runtime.c
crm_resource_LDADD = $(top_builddir)/lib/pengine/libpe_rules.la \
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 6853ad5..c8c1cfa 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -11,62 +11,70 @@
#include <pacemaker-internal.h>
#include <sys/param.h>
-
-#include <crm/crm.h>
-#include <crm/stonith-ng.h>
-
#include <stdio.h>
#include <sys/types.h>
#include <unistd.h>
-
#include <stdlib.h>
#include <errno.h>
#include <fcntl.h>
#include <libgen.h>
#include <time.h>
+#include <crm/crm.h>
+#include <crm/stonith-ng.h>
+#include <crm/common/ipc_controld.h>
+
bool BE_QUIET = FALSE;
bool scope_master = FALSE;
int cib_options = cib_sync_call;
-
-static GMainLoop *mainloop = NULL;
+static crm_exit_t exit_code = CRM_EX_OK;
// Things that should be cleaned up on exit
+static GMainLoop *mainloop = NULL;
static cib_t *cib_conn = NULL;
-static pcmk_controld_api_t *controld_api = NULL;
+static pcmk_ipc_api_t *controld_api = NULL;
static pe_working_set_t *data_set = NULL;
#define MESSAGE_TIMEOUT_S 60
// Clean up and exit
static crm_exit_t
-bye(crm_exit_t exit_code)
+bye(crm_exit_t ec)
{
- static bool crm_resource_shutdown_flag = false;
-
- if (crm_resource_shutdown_flag) {
- // Allow usage like "return bye(...);"
- return exit_code;
- }
- crm_resource_shutdown_flag = true;
-
if (cib_conn != NULL) {
cib_t *save_cib_conn = cib_conn;
- cib_conn = NULL;
+ cib_conn = NULL; // Ensure we can't free this twice
save_cib_conn->cmds->signoff(save_cib_conn);
cib_delete(save_cib_conn);
}
if (controld_api != NULL) {
- pcmk_controld_api_t *save_controld_api = controld_api;
+ pcmk_ipc_api_t *save_controld_api = controld_api;
- controld_api = NULL;
- pcmk_free_controld_api(save_controld_api);
+ controld_api = NULL; // Ensure we can't free this twice
+ pcmk_free_ipc_api(save_controld_api);
+ }
+ if (mainloop != NULL) {
+ g_main_loop_unref(mainloop);
+ mainloop = NULL;
}
pe_free_working_set(data_set);
data_set = NULL;
- crm_exit(exit_code);
- return exit_code;
+ crm_exit(ec);
+ return ec;
+}
+
+static void
+quit_main_loop(crm_exit_t ec)
+{
+ exit_code = ec;
+ if (mainloop != NULL) {
+ GMainLoop *mloop = mainloop;
+
+ mainloop = NULL; // Don't re-enter this block
+ pcmk_quit_main_loop(mloop, 10);
+ g_main_loop_unref(mloop);
+ }
}
static gboolean
@@ -76,39 +84,54 @@ resource_ipc_timeout(gpointer data)
fprintf(stderr, "\nAborting because no messages received in %d seconds\n",
MESSAGE_TIMEOUT_S);
crm_err("No messages received in %d seconds", MESSAGE_TIMEOUT_S);
- bye(CRM_EX_TIMEOUT);
+ quit_main_loop(CRM_EX_TIMEOUT);
return FALSE;
}
static void
-handle_controller_reply(pcmk_controld_api_t *capi, void *api_data,
- void *user_data)
+controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
+ crm_exit_t status, void *event_data, void *user_data)
{
- fprintf(stderr, ".");
- if ((capi->replies_expected(capi) == 0)
- && mainloop && g_main_loop_is_running(mainloop)) {
- fprintf(stderr, " OK\n");
- crm_debug("Got all the replies we expected");
- bye(CRM_EX_OK);
- }
-}
+ switch (event_type) {
+ case pcmk_ipc_event_disconnect:
+ if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
+ crm_info("Connection to controller was terminated");
+ }
+ quit_main_loop(exit_code);
+ break;
-static void
-handle_controller_drop(pcmk_controld_api_t *capi, void *api_data,
- void *user_data)
-{
- crm_info("Connection to controller was terminated");
- bye(CRM_EX_DISCONNECT);
+ case pcmk_ipc_event_reply:
+ if (status != CRM_EX_OK) {
+ fprintf(stderr, "\nError: bad reply from controller: %s\n",
+ crm_exit_str(status));
+ pcmk_disconnect_ipc(api);
+ quit_main_loop(status);
+ } else {
+ fprintf(stderr, ".");
+ if ((pcmk_controld_api_replies_expected(api) == 0)
+ && mainloop && g_main_loop_is_running(mainloop)) {
+ fprintf(stderr, " OK\n");
+ crm_debug("Got all the replies we expected");
+ pcmk_disconnect_ipc(api);
+ quit_main_loop(CRM_EX_OK);
+ }
+ }
+ break;
+
+ default:
+ break;
+ }
}
static void
-start_mainloop(pcmk_controld_api_t *capi)
+start_mainloop(pcmk_ipc_api_t *capi)
{
- if (capi->replies_expected(capi) > 0) {
- unsigned int count = capi->replies_expected(capi);
+ unsigned int count = pcmk_controld_api_replies_expected(capi);
+ if (count > 0) {
fprintf(stderr, "Waiting for %d %s from the controller",
count, pcmk__plural_alt(count, "reply", "replies"));
+ exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
mainloop = g_main_loop_new(NULL, FALSE);
g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
g_main_loop_run(mainloop);
@@ -664,7 +687,6 @@ main(int argc, char **argv)
int argerr = 0;
int flag;
int find_flags = 0; // Flags to use when searching for resource
- crm_exit_t exit_code = CRM_EX_OK;
crm_log_cli_init("crm_resource");
pcmk__set_cli_options(NULL, "<query>|<command> [options]", long_options,
@@ -1151,21 +1173,15 @@ main(int argc, char **argv)
// Establish a connection to the controller if needed
if (require_crmd) {
- char *client_uuid;
- pcmk_controld_api_cb_t dispatch_cb = {
- handle_controller_reply, NULL
- };
- pcmk_controld_api_cb_t destroy_cb = {
- handle_controller_drop, NULL
- };
-
-
- client_uuid = pcmk__getpid_s();
- controld_api = pcmk_new_controld_api(crm_system_name, client_uuid);
- free(client_uuid);
-
- rc = controld_api->connect(controld_api, true, &dispatch_cb,
- &destroy_cb);
+ rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
+ if (rc != pcmk_rc_ok) {
+ CMD_ERR("Error connecting to the controller: %s", pcmk_rc_str(rc));
+ rc = pcmk_rc2legacy(rc);
+ goto bail;
+ }
+ pcmk_register_ipc_callback(controld_api, controller_event_callback,
+ NULL);
+ rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main);
if (rc != pcmk_rc_ok) {
CMD_ERR("Error connecting to the controller: %s", pcmk_rc_str(rc));
rc = pcmk_rc2legacy(rc);
@@ -1525,8 +1541,8 @@ main(int argc, char **argv)
NULL, NULL, NULL,
NULL, attr_options));
- if (controld_api->reprobe(controld_api, host_uname,
- router_node) == pcmk_rc_ok) {
+ if (pcmk_controld_api_reprobe(controld_api, host_uname,
+ router_node) == pcmk_rc_ok) {
start_mainloop(controld_api);
}
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
index 0bf7bee..cb7f506 100644
--- a/tools/crm_resource.h
+++ b/tools/crm_resource.h
@@ -21,7 +21,6 @@
#include <crm/pengine/status.h>
#include <crm/pengine/internal.h>
#include <pacemaker-internal.h>
-#include "crm_resource_controller.h"
extern bool print_pending;
@@ -36,8 +35,6 @@ extern char *move_lifetime;
extern const char *attr_set_type;
-extern pcmk_controld_api_cb_t controld_api_cb;
-
/* ban */
int cli_resource_prefer(const char *rsc_id, const char *host, cib_t * cib_conn);
int cli_resource_ban(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn);
@@ -63,16 +60,16 @@ int cli_resource_print_operations(const char *rsc_id, const char *host_uname, bo
/* runtime */
void cli_resource_check(cib_t * cib, pe_resource_t *rsc);
-int cli_resource_fail(pcmk_controld_api_t *controld_api,
+int cli_resource_fail(pcmk_ipc_api_t *controld_api,
const char *host_uname, const char *rsc_id,
pe_working_set_t *data_set);
int cli_resource_search(pe_resource_t *rsc, const char *requested_name,
pe_working_set_t *data_set);
-int cli_resource_delete(pcmk_controld_api_t *controld_api,
+int cli_resource_delete(pcmk_ipc_api_t *controld_api,
const char *host_uname, pe_resource_t *rsc,
const char *operation, const char *interval_spec,
bool just_failures, pe_working_set_t *data_set);
-int cli_cleanup_all(pcmk_controld_api_t *controld_api, const char *node_name,
+int cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
const char *operation, const char *interval_spec,
pe_working_set_t *data_set);
int cli_resource_restart(pe_resource_t *rsc, const char *host, int timeout_ms,
diff --git a/tools/crm_resource_controller.c b/tools/crm_resource_controller.c
deleted file mode 100644
index 994a7be..0000000
--- a/tools/crm_resource_controller.c
+++ /dev/null
@@ -1,425 +0,0 @@
-/*
- * Copyright 2020 the Pacemaker project contributors
- *
- * The version control history for this file may have further details.
- *
- * This source code is licensed under the GNU General Public License version 2
- * or later (GPLv2+) WITHOUT ANY WARRANTY.
- */
-
-#include <crm_internal.h>
-#include <stdio.h>
-#include <errno.h>
-#include "crm_resource.h"
-
-// API object's private members
-struct controller_private {
- char *client_name; // Client name to use with IPC
- char *client_uuid; // Client UUID to use with IPC
- mainloop_io_t *source; // If main loop used, I/O source for IPC
- crm_ipc_t *ipc; // IPC connection to controller
- int replies_expected; // How many controller replies are expected
- pcmk_controld_api_cb_t dispatch_cb; // Caller's registered dispatch callback
- pcmk_controld_api_cb_t destroy_cb; // Caller's registered destroy callback
-};
-
-static void
-call_client_callback(pcmk_controld_api_t *api, pcmk_controld_api_cb_t *cb,
- void *api_data)
-{
- if ((cb != NULL) && (cb->callback != NULL)) {
- cb->callback(api, api_data, cb->user_data);
- }
-}
-
-/*
- * IPC callbacks when used with main loop
- */
-
-static void
-controller_ipc_destroy(gpointer user_data)
-{
- pcmk_controld_api_t *api = user_data;
- struct controller_private *private = api->private;
-
- private->ipc = NULL;
- private->source = NULL;
- call_client_callback(api, &(private->destroy_cb), NULL);
-}
-
-// \return < 0 if connection is no longer required, >= 0 if it is
-static int
-controller_ipc_dispatch(const char *buffer, ssize_t length, gpointer user_data)
-{
- xmlNode *msg = NULL;
- pcmk_controld_api_t *api = user_data;
-
- CRM_CHECK(buffer && api && api->private, return 0);
-
- msg = string2xml(buffer);
- if (msg == NULL) {
- crm_warn("Received malformed controller IPC message");
- } else {
- struct controller_private *private = api->private;
-
- crm_log_xml_trace(msg, "controller-reply");
- private->replies_expected--;
- call_client_callback(api, &(private->dispatch_cb),
- get_message_xml(msg, F_CRM_DATA));
- free_xml(msg);
- }
- return 0;
-}
-
-/*
- * IPC utilities
- */
-
-// \return Standard Pacemaker return code
-static int
-send_hello(crm_ipc_t *ipc, const char *client_name, const char *client_uuid)
-{
- xmlNode *hello = create_hello_message(client_uuid, client_name, "0", "1");
- int rc = crm_ipc_send(ipc, hello, 0, 0, NULL);
-
- free_xml(hello);
- if (rc < 0) {
- rc = pcmk_legacy2rc(rc);
- crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s",
- CRM_SYSTEM_CRMD /* ipc->name */,
- pcmk_rc_str(rc), rc);
- return rc;
- }
- crm_debug("Sent IPC hello to %s", CRM_SYSTEM_CRMD /* ipc->name */);
- return pcmk_rc_ok;
-}
-
-// \return Standard Pacemaker return code
-static int
-send_controller_request(pcmk_controld_api_t *api, const char *op,
- xmlNode *msg_data, const char *node)
-{
- int rc;
- struct controller_private *private = api->private;
- xmlNode *cmd = create_request(op, msg_data, node, CRM_SYSTEM_CRMD,
- private->client_name, private->client_uuid);
- const char *reference = crm_element_value(cmd, XML_ATTR_REFERENCE);
-
- if ((cmd == NULL) || (reference == NULL)) {
- return EINVAL;
- }
-
- //@TODO pass as args? 0=crm_ipc_flags, 0=timeout_ms (default 5s), NULL=reply
- crm_log_xml_trace(cmd, "controller-request");
- rc = crm_ipc_send(private->ipc, cmd, 0, 0, NULL);
- free_xml(cmd);
- if (rc < 0) {
- return pcmk_legacy2rc(rc);
- }
- private->replies_expected++;
- return pcmk_rc_ok;
-}
-
-/*
- * pcmk_controld_api_t methods
- */
-
-static int
-controller_connect_mainloop(pcmk_controld_api_t *api)
-{
- struct controller_private *private = api->private;
- struct ipc_client_callbacks callbacks = {
- .dispatch = controller_ipc_dispatch,
- .destroy = controller_ipc_destroy,
- };
-
- private->source = mainloop_add_ipc_client(CRM_SYSTEM_CRMD,
- G_PRIORITY_DEFAULT, 0, api,
- &callbacks);
- if (private->source == NULL) {
- return ENOTCONN;
- }
-
- private->ipc = mainloop_get_ipc_client(private->source);
- if (private->ipc == NULL) {
- (void) api->disconnect(api);
- return ENOTCONN;
- }
-
- crm_debug("Connected to %s IPC (attaching to main loop)", CRM_SYSTEM_CRMD);
- return pcmk_rc_ok;
-}
-
-static int
-controller_connect_no_mainloop(pcmk_controld_api_t *api)
-{
- struct controller_private *private = api->private;
-
- private->ipc = crm_ipc_new(CRM_SYSTEM_CRMD, 0);
- if (private->ipc == NULL) {
- return ENOTCONN;
- }
- if (!crm_ipc_connect(private->ipc)) {
- crm_ipc_close(private->ipc);
- crm_ipc_destroy(private->ipc);
- private->ipc = NULL;
- return errno;
- }
- /* @TODO caller needs crm_ipc_get_fd(private->ipc); either add method for
- * that, or replace use_mainloop with int *fd
- */
- crm_debug("Connected to %s IPC", CRM_SYSTEM_CRMD);
- return pcmk_rc_ok;
-}
-
-static void
-set_callback(pcmk_controld_api_cb_t *dest, pcmk_controld_api_cb_t *source)
-{
- if (source) {
- dest->callback = source->callback;
- dest->user_data = source->user_data;
- }
-}
-
-static int
-controller_api_connect(pcmk_controld_api_t *api, bool use_mainloop,
- pcmk_controld_api_cb_t *dispatch_cb,
- pcmk_controld_api_cb_t *destroy_cb)
-{
- int rc = pcmk_rc_ok;
- struct controller_private *private;
-
- if (api == NULL) {
- return EINVAL;
- }
- private = api->private;
-
- set_callback(&(private->dispatch_cb), dispatch_cb);
- set_callback(&(private->destroy_cb), destroy_cb);
-
- if (private->ipc != NULL) {
- return pcmk_rc_ok; // already connected
- }
-
- if (use_mainloop) {
- rc = controller_connect_mainloop(api);
- } else {
- rc = controller_connect_no_mainloop(api);
- }
- if (rc != pcmk_rc_ok) {
- return rc;
- }
-
- rc = send_hello(private->ipc, private->client_name, private->client_uuid);
- if (rc != pcmk_rc_ok) {
- (void) api->disconnect(api);
- }
- return rc;
-}
-
-static int
-controller_api_disconnect(pcmk_controld_api_t *api)
-{
- struct controller_private *private = api->private;
-
- if (private->source != NULL) {
- // Attached to main loop
- mainloop_del_ipc_client(private->source);
- private->source = NULL;
- private->ipc = NULL;
-
- } else if (private->ipc != NULL) {
- // Not attached to main loop
- crm_ipc_t *ipc = private->ipc;
-
- private->ipc = NULL;
- crm_ipc_close(ipc);
- crm_ipc_destroy(ipc);
- }
- crm_debug("Disconnected from %s IPC", CRM_SYSTEM_CRMD /* ipc->name */);
- return pcmk_rc_ok;
-}
-
-//@TODO dispatch function for non-mainloop a la stonith_dispatch()
-//@TODO convenience retry-connect function a la stonith_api_connect_retry()
-
-static unsigned int
-controller_api_replies_expected(pcmk_controld_api_t *api)
-{
- if (api != NULL) {
- struct controller_private *private = api->private;
-
- return private->replies_expected;
- }
- return 0;
-}
-
-static xmlNode *
-create_reprobe_message_data(const char *target_node, const char *router_node)
-{
- xmlNode *msg_data;
-
- msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE);
- crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
- if ((router_node != NULL) && safe_str_neq(router_node, target_node)) {
- crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
- }
- return msg_data;
-}
-
-static int
-controller_api_reprobe(pcmk_controld_api_t *api, const char *target_node,
- const char *router_node)
-{
- int rc = EINVAL;
-
- if (api != NULL) {
- xmlNode *msg_data;
-
- crm_debug("Sending %s IPC request to reprobe %s via %s",
- CRM_SYSTEM_CRMD, crm_str(target_node), crm_str(router_node));
- msg_data = create_reprobe_message_data(target_node, router_node);
- rc = send_controller_request(api, CRM_OP_REPROBE, msg_data,
- (router_node? router_node : target_node));
- free_xml(msg_data);
- }
- return rc;
-}
-
-// \return Standard Pacemaker return code
-static int
-controller_resource_op(pcmk_controld_api_t *api, const char *op,
- const char *target_node, const char *router_node,
- bool cib_only, const char *rsc_id,
- const char *rsc_long_id, const char *standard,
- const char *provider, const char *type)
-{
- int rc;
- char *key;
- xmlNode *msg_data, *xml_rsc, *params;
-
- if (api == NULL) {
- return EINVAL;
- }
- if (router_node == NULL) {
- router_node = target_node;
- }
-
- msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
-
- /* The controller logs the transition key from resource op requests, so we
- * need to have *something* for it.
- */
- key = pcmk__transition_key(0, getpid(), 0,
- "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
- crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
- free(key);
-
- crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
- if (safe_str_neq(router_node, target_node)) {
- crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
- }
-
- if (cib_only) {
- // Indicate that only the CIB needs to be cleaned
- crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB);
- }
-
- xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
- crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
- crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc_long_id);
- crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, standard);
- crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, provider);
- crm_xml_add(xml_rsc, XML_ATTR_TYPE, type);
-
- params = create_xml_node(msg_data, XML_TAG_ATTRS);
- crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
-
- // The controller parses the timeout from the request
- key = crm_meta_name(XML_ATTR_TIMEOUT);
- crm_xml_add(params, key, "60000"); /* 1 minute */ //@TODO pass as arg
- free(key);
-
- rc = send_controller_request(api, op, msg_data, router_node);
- free_xml(msg_data);
- return rc;
-}
-
-static int
-controller_api_fail_resource(pcmk_controld_api_t *api,
- const char *target_node, const char *router_node,
- const char *rsc_id, const char *rsc_long_id,
- const char *standard, const char *provider,
- const char *type)
-{
- crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s",
- CRM_SYSTEM_CRMD, crm_str(rsc_id), crm_str(rsc_long_id),
- crm_str(target_node), crm_str(router_node));
- return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node,
- router_node, false, rsc_id, rsc_long_id,
- standard, provider, type);
-}
-
-static int
-controller_api_refresh_resource(pcmk_controld_api_t *api,
- const char *target_node,
- const char *router_node,
- const char *rsc_id, const char *rsc_long_id,
- const char *standard, const char *provider,
- const char *type, bool cib_only)
-{
- crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s",
- CRM_SYSTEM_CRMD, crm_str(rsc_id), crm_str(rsc_long_id),
- crm_str(target_node), crm_str(router_node));
- return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node,
- router_node, cib_only, rsc_id, rsc_long_id,
- standard, provider, type);
-}
-
-pcmk_controld_api_t *
-pcmk_new_controld_api(const char *client_name, const char *client_uuid)
-{
- struct controller_private *private;
- pcmk_controld_api_t *api = calloc(1, sizeof(pcmk_controld_api_t));
-
- CRM_ASSERT(api != NULL);
-
- api->private = calloc(1, sizeof(struct controller_private));
- CRM_ASSERT(api->private != NULL);
- private = api->private;
-
- if (client_name == NULL) {
- client_name = crm_system_name? crm_system_name : "client";
- }
- private->client_name = strdup(client_name);
- CRM_ASSERT(private->client_name != NULL);
-
- if (client_uuid == NULL) {
- private->client_uuid = crm_generate_uuid();
- } else {
- private->client_uuid = strdup(client_uuid);
- }
- CRM_ASSERT(private->client_uuid != NULL);
-
- api->connect = controller_api_connect;
- api->disconnect = controller_api_disconnect;
- api->replies_expected = controller_api_replies_expected;
- api->reprobe = controller_api_reprobe;
- api->fail_resource = controller_api_fail_resource;
- api->refresh_resource = controller_api_refresh_resource;
- return api;
-}
-
-void
-pcmk_free_controld_api(pcmk_controld_api_t *api)
-{
- if (api != NULL) {
- struct controller_private *private = api->private;
-
- api->disconnect(api);
- free(private->client_name);
- free(private->client_uuid);
- free(api->private);
- free(api);
- }
-}
diff --git a/tools/crm_resource_controller.h b/tools/crm_resource_controller.h
deleted file mode 100644
index 50e20b4..0000000
--- a/tools/crm_resource_controller.h
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Copyright 2020 the Pacemaker project contributors
- *
- * The version control history for this file may have further details.
- *
- * This source code is licensed under the GNU Lesser General Public License
- * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
- */
-#ifndef PCMK__CONTROLD_API_H
-#define PCMK__CONTROLD_API_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdbool.h> // bool
-
-/* This is a demonstration of an abstracted controller IPC API. It is expected
- * that this will be improved and moved to libcrmcommon.
- *
- * @TODO We could consider whether it's reasonable to have a single type for
- * all daemons' IPC APIs (e.g. pcmk_ipc_api_t instead of pcmk_*_api_t). They
- * could potentially have common connect/disconnect methods and then a void* to
- * a group of API-specific methods.
- *
- * In that case, the callback type would also need to be generic, taking
- * (pcmk_ipc_api_t *api, void *api_data, void *user_data), with individual APIs
- * having functions for getting useful info from api_data. If all APIs followed
- * the call_id model, we could use int call_id instead of api_data.
- *
- * A major annoyance is that the controller IPC protocol currently does not have
- * any way to tie a particular reply to a particular request. The current
- * clients (crmadmin, crm_node, and crm_resource) simply know what kind of reply
- * to expect for the kind of request they sent. In crm_resource's case, all it
- * does is count replies, ignoring their content altogether.
- *
- * That really forces us to have a single callback for all events rather than a
- * per-request callback. That in turn implies that callers can only provide a
- * single user data pointer.
- *
- * @TODO Define protocol version constants to use in hello message.
- * @TODO Allow callers to specify timeouts.
- * @TODO Define call IDs for controller ops, while somehow maintaining backward
- * compatibility, since a client running on a Pacemaker Remote node could
- * be older or newer than the controller on the connection's cluster
- * node.
- * @TODO The controller currently does not respond to hello messages. We should
- * establish a common connection handshake protocol for all daemons that
- * involves a hello message and acknowledgement. We should support sync
- * or async connection (i.e. block until the ack is received, or return
- * after the hello is sent and call a connection callback when the hello
- * ack is received).
- */
-
-//! \internal
-typedef struct pcmk_controld_api_s pcmk_controld_api_t;
-
-//! \internal
-typedef struct pcmk_controld_api_callback_s {
- void (*callback)(pcmk_controld_api_t *api, void *api_data, void *user_data);
- void *user_data;
-} pcmk_controld_api_cb_t;
-
-//! \internal
-struct pcmk_controld_api_s {
- //! \internal
- void *private;
-
- /*!
- * \internal
- * \brief Connect to the local controller
- *
- * \param[in] api Controller API instance
- * \param[in] use_mainloop If true, attach IPC to main loop
- * \param[in] dispatch_cb If not NULL, call this when replies are received
- * \param[in] destroy_cb If not NULL, call this if connection drops
- *
- * \return Standard Pacemaker return code
- * \note Only the pointers inside the callback objects need to be
- * persistent, not the callback objects themselves. The destroy_cb
- * will be called only for unrequested disconnects.
- */
- int (*connect)(pcmk_controld_api_t *api, bool use_mainloop,
- pcmk_controld_api_cb_t *dispatch_cb,
- pcmk_controld_api_cb_t *destroy_cb);
-
- /*!
- * \internal
- * \brief Disconnect from the local controller
- *
- * \param[in] api Controller API instance
- *
- * \return Standard Pacemaker return code
- */
- int (*disconnect)(pcmk_controld_api_t *api);
-
- /*!
- * \internal
- * \brief Check number of replies still expected from controller
- *
- * \param[in] api Controller API instance
- *
- * \return Number of expected replies
- */
- unsigned int (*replies_expected)(pcmk_controld_api_t *api);
-
- /*!
- * \internal
- * \brief Send a reprobe controller operation
- *
- * \param[in] api Controller API instance
- * \param[in] target_node Name of node to reprobe
- * \param[in] router_node Router node for host
- *
- * \return Standard Pacemaker return code
- */
- int (*reprobe)(pcmk_controld_api_t *api, const char *target_node,
- const char *router_node);
-
- /* @TODO These methods have a lot of arguments. One possibility would be to
- * make a struct for agent info (standard/provider/type), which theortically
- * could be used throughout pacemaker code. However that would end up being
- * really awkward to use generically, since sometimes you need to allocate
- * those strings (char *) and other times you only have references into XML
- * (const char *). We could make some structs just for this API.
- */
-
- /*!
- * \internal
- * \brief Ask the controller to fail a resource
- *
- * \param[in] api Controller API instance
- * \param[in] target_node Name of node resource is on
- * \param[in] router_node Router node for target
- * \param[in] rsc_id ID of resource to fail
- * \param[in] rsc_long_id Long ID of resource (if any)
- * \param[in] standard Standard of resource
- * \param[in] provider Provider of resource (if any)
- * \param[in] type Type of resource to fail
- *
- * \return Standard Pacemaker return code
- */
- int (*fail_resource)(pcmk_controld_api_t *api, const char *target_node,
- const char *router_node, const char *rsc_id,
- const char *rsc_long_id, const char *standard,
- const char *provider, const char *type);
-
- /*!
- * \internal
- * \brief Ask the controller to refresh a resource
- *
- * \param[in] api Controller API instance
- * \param[in] target_node Name of node resource is on
- * \param[in] router_node Router node for target
- * \param[in] rsc_id ID of resource to refresh
- * \param[in] rsc_long_id Long ID of resource (if any)
- * \param[in] standard Standard of resource
- * \param[in] provider Provider of resource (if any)
- * \param[in] type Type of resource
- * \param[in] cib_only If true, clean resource from CIB only
- *
- * \return Standard Pacemaker return code
- */
- int (*refresh_resource)(pcmk_controld_api_t *api, const char *target_node,
- const char *router_node, const char *rsc_id,
- const char *rsc_long_id, const char *standard,
- const char *provider, const char *type,
- bool cib_only);
-};
-
-/*!
- * \internal
- * \brief Create new controller IPC API object for clients
- *
- * \param[in] client_name Client name to use with IPC
- * \param[in] client_uuid Client UUID to use with IPC
- *
- * \return Newly allocated object
- * \note This function asserts on errors, so it will never return NULL.
- * The caller is responsible for freeing the result with
- * pcmk_free_controld_api().
- */
-pcmk_controld_api_t *pcmk_new_controld_api(const char *client_name,
- const char *client_uuid);
-
-/*!
- * \internal
- * \brief Free a controller IPC API object
- *
- * \param[in] api Controller IPC API object to free
- */
-void pcmk_free_controld_api(pcmk_controld_api_t *api);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 37789d1..cc2abeb 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -8,6 +8,7 @@
*/
#include <crm_resource.h>
+#include <crm/common/ipc_controld.h>
int resource_verbose = 0;
bool do_force = FALSE;
@@ -460,7 +461,7 @@ cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
// \return Standard Pacemaker return code
static int
-send_lrm_rsc_op(pcmk_controld_api_t *controld_api, bool do_fail_resource,
+send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
const char *host_uname, const char *rsc_id,
pe_working_set_t *data_set)
{
@@ -528,14 +529,13 @@ send_lrm_rsc_op(pcmk_controld_api_t *controld_api, bool do_fail_resource,
rsc_api_id = rsc->id;
}
if (do_fail_resource) {
- return controld_api->fail_resource(controld_api, host_uname,
- router_node, rsc_api_id, rsc_long_id,
- rsc_class, rsc_provider, rsc_type);
+ return pcmk_controld_api_fail(controld_api, host_uname, router_node,
+ rsc_api_id, rsc_long_id,
+ rsc_class, rsc_provider, rsc_type);
} else {
- return controld_api->refresh_resource(controld_api, host_uname,
- router_node, rsc_api_id,
- rsc_long_id, rsc_class,
- rsc_provider, rsc_type, cib_only);
+ return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
+ rsc_api_id, rsc_long_id, rsc_class,
+ rsc_provider, rsc_type, cib_only);
}
}
@@ -558,7 +558,7 @@ rsc_fail_name(pe_resource_t *rsc)
// \return Standard Pacemaker return code
static int
-clear_rsc_history(pcmk_controld_api_t *controld_api, const char *host_uname,
+clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
const char *rsc_id, pe_working_set_t *data_set)
{
int rc = pcmk_ok;
@@ -574,16 +574,16 @@ clear_rsc_history(pcmk_controld_api_t *controld_api, const char *host_uname,
}
crm_trace("Processing %d mainloop inputs",
- controld_api->replies_expected(controld_api));
+ pcmk_controld_api_replies_expected(controld_api));
while (g_main_context_iteration(NULL, FALSE)) {
crm_trace("Processed mainloop input, %d still remaining",
- controld_api->replies_expected(controld_api));
+ pcmk_controld_api_replies_expected(controld_api));
}
return rc;
}
static int
-clear_rsc_failures(pcmk_controld_api_t *controld_api, const char *node_name,
+clear_rsc_failures(pcmk_ipc_api_t *controld_api, const char *node_name,
const char *rsc_id, const char *operation,
const char *interval_spec, pe_working_set_t *data_set)
{
@@ -683,7 +683,7 @@ clear_rsc_fail_attrs(pe_resource_t *rsc, const char *operation,
}
int
-cli_resource_delete(pcmk_controld_api_t *controld_api, const char *host_uname,
+cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
pe_resource_t *rsc, const char *operation,
const char *interval_spec, bool just_failures,
pe_working_set_t *data_set)
@@ -792,7 +792,7 @@ cli_resource_delete(pcmk_controld_api_t *controld_api, const char *host_uname,
}
int
-cli_cleanup_all(pcmk_controld_api_t *controld_api, const char *node_name,
+cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
const char *operation, const char *interval_spec,
pe_working_set_t *data_set)
{
@@ -905,7 +905,7 @@ cli_resource_check(cib_t * cib_conn, pe_resource_t *rsc)
// \return Standard Pacemaker return code
int
-cli_resource_fail(pcmk_controld_api_t *controld_api, const char *host_uname,
+cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
const char *rsc_id, pe_working_set_t *data_set)
{
crm_notice("Failing %s on %s", rsc_id, host_uname);
--
1.8.3.1
From ae14fa4a831e45eae0d78b0f42765fcf40c4ce56 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 14 Apr 2020 14:06:02 -0500
Subject: [PATCH 4/6] Refactor: tools: convert crm_node to use new controller
IPC model
---
tools/crm_node.c | 281 +++++++++++++++++++++++++++----------------------------
1 file changed, 140 insertions(+), 141 deletions(-)
diff --git a/tools/crm_node.c b/tools/crm_node.c
index db31f20..1773a36 100644
--- a/tools/crm_node.c
+++ b/tools/crm_node.c
@@ -19,6 +19,7 @@
#include <crm/common/mainloop.h>
#include <crm/msg_xml.h>
#include <crm/cib.h>
+#include <crm/common/ipc_controld.h>
#include <crm/common/attrd_internal.h>
#define SUMMARY "crm_node - Tool for displaying low-level node information"
@@ -39,7 +40,6 @@ gboolean command_cb(const gchar *option_name, const gchar *optarg, gpointer data
gboolean name_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
gboolean remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
-static char *pid_s = NULL;
static GMainLoop *mainloop = NULL;
static crm_exit_t exit_code = CRM_EX_OK;
@@ -140,11 +140,6 @@ remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError *
static void
crm_node_exit(crm_exit_t value)
{
- if (pid_s) {
- free(pid_s);
- pid_s = NULL;
- }
-
exit_code = value;
if (mainloop && g_main_loop_is_running(mainloop)) {
@@ -155,173 +150,123 @@ crm_node_exit(crm_exit_t value)
}
static void
-exit_disconnect(gpointer user_data)
-{
- fprintf(stderr, "error: Lost connection to cluster\n");
- crm_node_exit(CRM_EX_DISCONNECT);
-}
-
-typedef int (*ipc_dispatch_fn) (const char *buffer, ssize_t length,
- gpointer userdata);
-
-static crm_ipc_t *
-new_mainloop_for_ipc(const char *system, ipc_dispatch_fn dispatch)
+controller_event_cb(pcmk_ipc_api_t *controld_api,
+ enum pcmk_ipc_event event_type, crm_exit_t status,
+ void *event_data, void *user_data)
{
- mainloop_io_t *source = NULL;
- crm_ipc_t *ipc = NULL;
-
- struct ipc_client_callbacks ipc_callbacks = {
- .dispatch = dispatch,
- .destroy = exit_disconnect
- };
+ pcmk_controld_api_reply_t *reply = event_data;
- mainloop = g_main_loop_new(NULL, FALSE);
- source = mainloop_add_ipc_client(system, G_PRIORITY_DEFAULT, 0,
- NULL, &ipc_callbacks);
- ipc = mainloop_get_ipc_client(source);
- if (ipc == NULL) {
- fprintf(stderr,
- "error: Could not connect to cluster (is it running?)\n");
- crm_node_exit(CRM_EX_DISCONNECT);
- }
- return ipc;
-}
-
-static int
-send_controller_hello(crm_ipc_t *controller)
-{
- xmlNode *hello = NULL;
- int rc;
-
- pid_s = pcmk__getpid_s();
- hello = create_hello_message(pid_s, crm_system_name, "1", "0");
- rc = crm_ipc_send(controller, hello, 0, 0, NULL);
- free_xml(hello);
- return (rc < 0)? rc : 0;
-}
-
-static int
-send_node_info_request(crm_ipc_t *controller, uint32_t nodeid)
-{
- xmlNode *ping = NULL;
- int rc;
-
- ping = create_request(CRM_OP_NODE_INFO, NULL, NULL, CRM_SYSTEM_CRMD,
- crm_system_name, pid_s);
- if (nodeid > 0) {
- crm_xml_add_int(ping, XML_ATTR_ID, nodeid);
- }
- rc = crm_ipc_send(controller, ping, 0, 0, NULL);
- free_xml(ping);
- return (rc < 0)? rc : 0;
-}
+ switch (event_type) {
+ case pcmk_ipc_event_disconnect:
+ if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
+ fprintf(stderr, "error: Lost connection to controller\n");
+ }
+ goto done;
+ break;
-static int
-dispatch_controller(const char *buffer, ssize_t length, gpointer userdata)
-{
- xmlNode *message = string2xml(buffer);
- xmlNode *data = NULL;
- const char *value = NULL;
+ case pcmk_ipc_event_reply:
+ break;
- if (message == NULL) {
- fprintf(stderr, "error: Could not understand reply from controller\n");
- crm_node_exit(CRM_EX_PROTOCOL);
- return 0;
+ default:
+ return;
}
- crm_log_xml_trace(message, "controller reply");
-
- exit_code = CRM_EX_PROTOCOL;
- // Validate reply
- value = crm_element_value(message, F_CRM_MSG_TYPE);
- if (safe_str_neq(value, XML_ATTR_RESPONSE)) {
- fprintf(stderr, "error: Message from controller was not a reply\n");
+ if (status != CRM_EX_OK) {
+ fprintf(stderr, "error: Bad reply from controller: %s\n",
+ crm_exit_str(status));
goto done;
}
- value = crm_element_value(message, XML_ATTR_REFERENCE);
- if (value == NULL) {
- fprintf(stderr, "error: Controller reply did not specify original message\n");
- goto done;
- }
- data = get_message_xml(message, F_CRM_DATA);
- if (data == NULL) {
- fprintf(stderr, "error: Controller reply did not contain any data\n");
+ if (reply->reply_type != pcmk_controld_reply_info) {
+ fprintf(stderr, "error: Unknown reply type %d from controller\n",
+ reply->reply_type);
goto done;
}
+ // Parse desired info from reply and display to user
switch (options.command) {
case 'i':
- value = crm_element_value(data, XML_ATTR_ID);
- if (value == NULL) {
- fprintf(stderr, "error: Controller reply did not contain node ID\n");
- } else {
- printf("%s\n", value);
- exit_code = CRM_EX_OK;
+ if (reply->data.node_info.id == 0) {
+ fprintf(stderr,
+ "error: Controller reply did not contain node ID\n");
+ exit_code = CRM_EX_PROTOCOL;
+ goto done;
}
+ printf("%d\n", reply->data.node_info.id);
break;
case 'n':
case 'N':
- value = crm_element_value(data, XML_ATTR_UNAME);
- if (value == NULL) {
+ if (reply->data.node_info.uname == NULL) {
fprintf(stderr, "Node is not known to cluster\n");
exit_code = CRM_EX_NOHOST;
- } else {
- printf("%s\n", value);
- exit_code = CRM_EX_OK;
+ goto done;
}
+ printf("%s\n", reply->data.node_info.uname);
break;
case 'q':
- value = crm_element_value(data, XML_ATTR_HAVE_QUORUM);
- if (value == NULL) {
- fprintf(stderr, "error: Controller reply did not contain quorum status\n");
- } else {
- bool quorum = crm_is_true(value);
-
- printf("%d\n", quorum);
- exit_code = quorum? CRM_EX_OK : CRM_EX_QUORUM;
+ printf("%d\n", reply->data.node_info.have_quorum);
+ if (!(reply->data.node_info.have_quorum)) {
+ exit_code = CRM_EX_QUORUM;
+ goto done;
}
break;
default:
fprintf(stderr, "internal error: Controller reply not expected\n");
exit_code = CRM_EX_SOFTWARE;
- break;
+ goto done;
}
+ // Success
+ exit_code = CRM_EX_OK;
done:
- free_xml(message);
- crm_node_exit(exit_code);
- return 0;
+ pcmk_disconnect_ipc(controld_api);
+ pcmk_quit_main_loop(mainloop, 10);
}
static void
run_controller_mainloop(uint32_t nodeid)
{
- crm_ipc_t *controller = NULL;
+ pcmk_ipc_api_t *controld_api = NULL;
int rc;
- controller = new_mainloop_for_ipc(CRM_SYSTEM_CRMD, dispatch_controller);
+ // Set disconnect exit code to handle unexpected disconnects
+ exit_code = CRM_EX_DISCONNECT;
+
+ // Create controller IPC object
+ rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
+ if (rc != pcmk_rc_ok) {
+ fprintf(stderr, "error: Could not connect to controller: %s\n",
+ pcmk_rc_str(rc));
+ return;
+ }
+ pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL);
- rc = send_controller_hello(controller);
- if (rc < 0) {
- fprintf(stderr, "error: Could not register with controller: %s\n",
- pcmk_strerror(rc));
- crm_node_exit(crm_errno2exit(rc));
+ // Connect to controller
+ rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main);
+ if (rc != pcmk_rc_ok) {
+ fprintf(stderr, "error: Could not connect to controller: %s\n",
+ pcmk_rc_str(rc));
+ exit_code = pcmk_rc2exitc(rc);
+ return;
}
- rc = send_node_info_request(controller, nodeid);
- if (rc < 0) {
+ rc = pcmk_controld_api_node_info(controld_api, nodeid);
+ if (rc != pcmk_rc_ok) {
fprintf(stderr, "error: Could not ping controller: %s\n",
- pcmk_strerror(rc));
- crm_node_exit(crm_errno2exit(rc));
+ pcmk_rc_str(rc));
+ pcmk_disconnect_ipc(controld_api);
+ exit_code = pcmk_rc2exitc(rc);
+ return;
}
- // Run main loop to get controller reply via dispatch_controller()
+ // Run main loop to get controller reply via controller_event_cb()
+ mainloop = g_main_loop_new(NULL, FALSE);
g_main_loop_run(mainloop);
g_main_loop_unref(mainloop);
mainloop = NULL;
+ pcmk_free_ipc_api(controld_api);
}
static void
@@ -385,32 +330,56 @@ cib_remove_node(long id, const char *name)
}
static int
+controller_remove_node(const char *node_name, long nodeid)
+{
+ pcmk_ipc_api_t *controld_api = NULL;
+ int rc;
+
+ // Create controller IPC object
+ rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
+ if (rc != pcmk_rc_ok) {
+ fprintf(stderr, "error: Could not connect to controller: %s\n",
+ pcmk_rc_str(rc));
+ return ENOTCONN;
+ }
+
+ // Connect to controller (without main loop)
+ rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_sync);
+ if (rc != pcmk_rc_ok) {
+ fprintf(stderr, "error: Could not connect to controller: %s\n",
+ pcmk_rc_str(rc));
+ pcmk_free_ipc_api(controld_api);
+ return rc;
+ }
+
+ rc = pcmk_ipc_purge_node(controld_api, node_name, nodeid);
+ if (rc != pcmk_rc_ok) {
+ fprintf(stderr,
+ "error: Could not clear node from controller's cache: %s\n",
+ pcmk_rc_str(rc));
+ }
+
+ pcmk_free_ipc_api(controld_api);
+ return pcmk_rc_ok;
+}
+
+static int
tools_remove_node_cache(const char *node_name, long nodeid, const char *target)
{
int rc = -1;
- crm_ipc_t *conn = crm_ipc_new(target, 0);
+ crm_ipc_t *conn = NULL;
xmlNode *cmd = NULL;
+ conn = crm_ipc_new(target, 0);
if (!conn) {
return -ENOTCONN;
}
-
if (!crm_ipc_connect(conn)) {
crm_perror(LOG_ERR, "Connection to %s failed", target);
crm_ipc_destroy(conn);
return -ENOTCONN;
}
- if(safe_str_eq(target, CRM_SYSTEM_CRMD)) {
- // The controller requires a hello message before sending a request
- rc = send_controller_hello(conn);
- if (rc < 0) {
- fprintf(stderr, "error: Could not register with controller: %s\n",
- pcmk_strerror(rc));
- return rc;
- }
- }
-
crm_trace("Removing %s[%ld] from the %s membership cache",
node_name, nodeid, target);
@@ -427,9 +396,9 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target)
crm_xml_add_int(cmd, PCMK__XA_ATTR_NODE_ID, (int) nodeid);
}
- } else {
- cmd = create_request(CRM_OP_RM_NODE_CACHE,
- NULL, NULL, target, crm_system_name, pid_s);
+ } else { // Fencer or pacemakerd
+ cmd = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, target,
+ crm_system_name, NULL);
if (nodeid > 0) {
crm_xml_set_id(cmd, "%ld", nodeid);
}
@@ -441,6 +410,7 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target)
target, node_name, nodeid, rc);
if (rc > 0) {
+ // @TODO Should this be done just once after all the rest?
rc = cib_remove_node(nodeid, node_name);
}
@@ -455,12 +425,12 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target)
static void
remove_node(const char *target_uname)
{
+ int rc;
int d = 0;
long nodeid = 0;
const char *node_name = NULL;
char *endptr = NULL;
const char *daemons[] = {
- CRM_SYSTEM_CRMD,
"stonith-ng",
T_ATTRD,
CRM_SYSTEM_MCP,
@@ -476,6 +446,12 @@ remove_node(const char *target_uname)
node_name = target_uname;
}
+ rc = controller_remove_node(node_name, nodeid);
+ if (rc != pcmk_rc_ok) {
+ exit_code = pcmk_rc2exitc(rc);
+ return;
+ }
+
for (d = 0; d < DIMOF(daemons); d++) {
if (tools_remove_node_cache(node_name, nodeid, daemons[d])) {
crm_err("Failed to connect to %s to remove node '%s'",
@@ -545,12 +521,34 @@ node_mcp_dispatch(const char *buffer, ssize_t length, gpointer userdata)
}
static void
+lost_pacemakerd(gpointer user_data)
+{
+ fprintf(stderr, "error: Lost connection to cluster\n");
+ exit_code = CRM_EX_DISCONNECT;
+ g_main_loop_quit(mainloop);
+}
+
+static void
run_pacemakerd_mainloop(void)
{
crm_ipc_t *ipc = NULL;
xmlNode *poke = NULL;
+ mainloop_io_t *source = NULL;
- ipc = new_mainloop_for_ipc(CRM_SYSTEM_MCP, node_mcp_dispatch);
+ struct ipc_client_callbacks ipc_callbacks = {
+ .dispatch = node_mcp_dispatch,
+ .destroy = lost_pacemakerd
+ };
+
+ source = mainloop_add_ipc_client(CRM_SYSTEM_MCP, G_PRIORITY_DEFAULT, 0,
+ NULL, &ipc_callbacks);
+ ipc = mainloop_get_ipc_client(source);
+ if (ipc == NULL) {
+ fprintf(stderr,
+ "error: Could not connect to cluster (is it running?)\n");
+ exit_code = CRM_EX_DISCONNECT;
+ return;
+ }
// Sending anything will get us a list of nodes
poke = create_xml_node(NULL, "poke");
@@ -558,6 +556,7 @@ run_pacemakerd_mainloop(void)
free_xml(poke);
// Handle reply via node_mcp_dispatch()
+ mainloop = g_main_loop_new(NULL, FALSE);
g_main_loop_run(mainloop);
g_main_loop_unref(mainloop);
mainloop = NULL;
--
1.8.3.1
From a361f764cb28630d440eec0f3e04a4f3812825eb Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 14 Apr 2020 16:05:15 -0500
Subject: [PATCH 5/6] Refactor: tools: remove dead code from crm_node
---
tools/crm_node.c | 22 +---------------------
1 file changed, 1 insertion(+), 21 deletions(-)
diff --git a/tools/crm_node.c b/tools/crm_node.c
index 1773a36..57c2ee5 100644
--- a/tools/crm_node.c
+++ b/tools/crm_node.c
@@ -130,25 +130,6 @@ remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError *
return TRUE;
}
-/*!
- * \internal
- * \brief Exit crm_node
- * Clean up memory, and either quit mainloop (if running) or exit
- *
- * \param[in] value Exit status
- */
-static void
-crm_node_exit(crm_exit_t value)
-{
- exit_code = value;
-
- if (mainloop && g_main_loop_is_running(mainloop)) {
- g_main_loop_quit(mainloop);
- } else {
- crm_exit(exit_code);
- }
-}
-
static void
controller_event_cb(pcmk_ipc_api_t *controld_api,
enum pcmk_ipc_event event_type, crm_exit_t status,
@@ -660,6 +641,5 @@ done:
g_strfreev(processed_args);
g_clear_error(&error);
pcmk__free_arg_context(context);
- crm_node_exit(exit_code);
- return exit_code;
+ return crm_exit(exit_code);
}
--
1.8.3.1
From 591944539259f6294450517770d95c7a02fc599c Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Mon, 20 Apr 2020 15:48:15 -0500
Subject: [PATCH 6/6] Refactor: tools: convert crmadmin to use new controller
IPC model
---
tools/crmadmin.c | 484 ++++++++++++++++++++++++-------------------------------
1 file changed, 208 insertions(+), 276 deletions(-)
diff --git a/tools/crmadmin.c b/tools/crmadmin.c
index 3e9e959..4688458 100644
--- a/tools/crmadmin.c
+++ b/tools/crmadmin.c
@@ -9,56 +9,44 @@
#include <crm_internal.h>
-#include <sys/param.h>
-
#include <stdio.h>
-#include <sys/types.h>
-#include <unistd.h>
+#include <stdbool.h>
+#include <stdlib.h> // atoi()
-#include <stdlib.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <libgen.h>
#include <glib.h> // gboolean, GMainLoop, etc.
+#include <libxml/tree.h> // xmlNode
#include <crm/crm.h>
+#include <crm/cib.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
-
+#include <crm/common/ipc_controld.h>
#include <crm/common/mainloop.h>
-#include <crm/cib.h>
-
#define DEFAULT_MESSAGE_TIMEOUT_MS 30000
static guint message_timer_id = 0;
static guint message_timeout_ms = DEFAULT_MESSAGE_TIMEOUT_MS;
static GMainLoop *mainloop = NULL;
-static crm_ipc_t *crmd_channel = NULL;
-static char *admin_uuid = NULL;
-
-gboolean do_init(void);
-int do_work(void);
-void crmadmin_ipc_connection_destroy(gpointer user_data);
-int admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata);
-int do_find_node_list(xmlNode * xml_node);
+
+bool do_work(pcmk_ipc_api_t *api);
+void do_find_node_list(xmlNode *xml_node);
gboolean admin_message_timeout(gpointer data);
+static enum {
+ cmd_none,
+ cmd_shutdown,
+ cmd_health,
+ cmd_elect_dc,
+ cmd_whois_dc,
+ cmd_list_nodes,
+} command = cmd_none;
+
static gboolean BE_VERBOSE = FALSE;
-static int expected_responses = 1;
static gboolean BASH_EXPORT = FALSE;
-static gboolean DO_HEALTH = FALSE;
-static gboolean DO_RESET = FALSE;
-static gboolean DO_RESOURCE = FALSE;
-static gboolean DO_ELECT_DC = FALSE;
-static gboolean DO_WHOIS_DC = FALSE;
-static gboolean DO_NODE_LIST = FALSE;
static gboolean BE_SILENT = FALSE;
-static gboolean DO_RESOURCE_LIST = FALSE;
-static const char *crmd_operation = NULL;
static char *dest_node = NULL;
static crm_exit_t exit_code = CRM_EX_OK;
-static const char *sys_to = NULL;
static pcmk__cli_option_t long_options[] = {
// long option, argument type, storage, short option, description, flags
@@ -133,7 +121,8 @@ static pcmk__cli_option_t long_options[] = {
},
{
"bash-export", no_argument, NULL, 'B',
- "Create Bash export entries of the form 'export uname=uuid'\n",
+ "Display nodes as shell commands of the form 'export uname=uuid' "
+ "(valid with -N/--nodes)'\n",
pcmk__option_default
},
{
@@ -142,13 +131,98 @@ static pcmk__cli_option_t long_options[] = {
},
{
"-spacer-", no_argument, NULL, '-',
- " The -K and -E commands are rarely used and may be removed in "
- "future versions.",
+ "The -K and -E commands do not work and may be removed in a future "
+ "version.",
pcmk__option_default
},
{ 0, 0, 0, 0 }
};
+static void
+quit_main_loop(crm_exit_t ec)
+{
+ exit_code = ec;
+ if (mainloop != NULL) {
+ GMainLoop *mloop = mainloop;
+
+ mainloop = NULL; // Don't re-enter this block
+ pcmk_quit_main_loop(mloop, 10);
+ g_main_loop_unref(mloop);
+ }
+}
+
+static void
+controller_event_cb(pcmk_ipc_api_t *controld_api,
+ enum pcmk_ipc_event event_type, crm_exit_t status,
+ void *event_data, void *user_data)
+{
+ pcmk_controld_api_reply_t *reply = event_data;
+
+ switch (event_type) {
+ case pcmk_ipc_event_disconnect:
+ if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
+ fprintf(stderr, "error: Lost connection to controller\n");
+ }
+ goto done;
+ break;
+
+ case pcmk_ipc_event_reply:
+ break;
+
+ default:
+ return;
+ }
+
+ if (message_timer_id != 0) {
+ g_source_remove(message_timer_id);
+ message_timer_id = 0;
+ }
+
+ if (status != CRM_EX_OK) {
+ fprintf(stderr, "error: Bad reply from controller: %s",
+ crm_exit_str(status));
+ exit_code = status;
+ goto done;
+ }
+
+ if (reply->reply_type != pcmk_controld_reply_ping) {
+ fprintf(stderr, "error: Unknown reply type %d from controller\n",
+ reply->reply_type);
+ goto done;
+ }
+
+ // Parse desired information from reply
+ switch (command) {
+ case cmd_health:
+ printf("Status of %s@%s: %s (%s)\n",
+ reply->data.ping.sys_from,
+ reply->host_from,
+ reply->data.ping.fsa_state,
+ reply->data.ping.result);
+ if (BE_SILENT && (reply->data.ping.fsa_state != NULL)) {
+ fprintf(stderr, "%s\n", reply->data.ping.fsa_state);
+ }
+ exit_code = CRM_EX_OK;
+ break;
+
+ case cmd_whois_dc:
+ printf("Designated Controller is: %s\n", reply->host_from);
+ if (BE_SILENT && (reply->host_from != NULL)) {
+ fprintf(stderr, "%s\n", reply->host_from);
+ }
+ exit_code = CRM_EX_OK;
+ break;
+
+ default: // Not really possible here
+ exit_code = CRM_EX_SOFTWARE;
+ break;
+ }
+
+done:
+ pcmk_disconnect_ipc(controld_api);
+ quit_main_loop(exit_code);
+}
+
// \return Standard Pacemaker return code
static int
list_nodes()
@@ -181,6 +255,9 @@ main(int argc, char **argv)
int option_index = 0;
int argerr = 0;
int flag;
+ int rc;
+ pcmk_ipc_api_t *controld_api = NULL;
+ bool need_controld_api = true;
crm_log_cli_init("crmadmin");
pcmk__set_cli_options(NULL, "<command> [options]", long_options,
@@ -211,33 +288,40 @@ main(int argc, char **argv)
pcmk__cli_help(flag, CRM_EX_OK);
break;
case 'D':
- DO_WHOIS_DC = TRUE;
+ command = cmd_whois_dc;
break;
case 'B':
BASH_EXPORT = TRUE;
break;
case 'K':
- DO_RESET = TRUE;
+ command = cmd_shutdown;
crm_trace("Option %c => %s", flag, optarg);
+ if (dest_node != NULL) {
+ free(dest_node);
+ }
dest_node = strdup(optarg);
- crmd_operation = CRM_OP_LOCAL_SHUTDOWN;
break;
case 'q':
BE_SILENT = TRUE;
break;
case 'S':
- DO_HEALTH = TRUE;
+ command = cmd_health;
crm_trace("Option %c => %s", flag, optarg);
+ if (dest_node != NULL) {
+ free(dest_node);
+ }
dest_node = strdup(optarg);
break;
case 'E':
- DO_ELECT_DC = TRUE;
+ command = cmd_elect_dc;
break;
case 'N':
- DO_NODE_LIST = TRUE;
+ command = cmd_list_nodes;
+ need_controld_api = false;
break;
case 'H':
- DO_HEALTH = TRUE;
+ fprintf(stderr, "Cluster-wide health option not supported\n");
+ ++argerr;
break;
default:
printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
@@ -257,285 +341,133 @@ main(int argc, char **argv)
++argerr;
}
+ if (command == cmd_none) {
+ fprintf(stderr, "error: Must specify a command option\n\n");
+ ++argerr;
+ }
+
if (argerr) {
pcmk__cli_help('?', CRM_EX_USAGE);
}
- if (do_init()) {
- int res = 0;
-
- res = do_work();
- if (res > 0) {
- /* wait for the reply by creating a mainloop and running it until
- * the callbacks are invoked...
- */
- mainloop = g_main_loop_new(NULL, FALSE);
- crm_trace("Waiting for %d replies from the local CRM", expected_responses);
-
- message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL);
-
- g_main_loop_run(mainloop);
-
- } else if (res < 0) {
- crm_err("No message to send");
- exit_code = CRM_EX_ERROR;
+ // Connect to the controller if needed
+ if (need_controld_api) {
+ rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
+ if (controld_api == NULL) {
+ fprintf(stderr, "error: Could not connect to controller: %s\n",
+ pcmk_rc_str(rc));
+ exit_code = pcmk_rc2exitc(rc);
+ goto done;
}
- } else {
- crm_warn("Init failed, could not perform requested operations");
- exit_code = CRM_EX_UNAVAILABLE;
- }
-
- crm_trace("%s exiting normally", crm_system_name);
- return exit_code;
-}
-
-int
-do_work(void)
-{
- int ret = 1;
-
- /* construct the request */
- xmlNode *msg_data = NULL;
- gboolean all_is_good = TRUE;
-
- if (DO_HEALTH == TRUE) {
- crm_trace("Querying the system");
-
- sys_to = CRM_SYSTEM_DC;
-
- if (dest_node != NULL) {
- sys_to = CRM_SYSTEM_CRMD;
- crmd_operation = CRM_OP_PING;
-
- if (BE_VERBOSE) {
- expected_responses = 1;
- }
-
- } else {
- crm_info("Cluster-wide health not available yet");
- all_is_good = FALSE;
+ pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL);
+ rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main);
+ if (rc != pcmk_rc_ok) {
+ fprintf(stderr, "error: Could not connect to controller: %s\n",
+ pcmk_rc_str(rc));
+ exit_code = pcmk_rc2exitc(rc);
+ goto done;
}
-
- } else if (DO_ELECT_DC) {
- /* tell the local node to initiate an election */
-
- dest_node = NULL;
- sys_to = CRM_SYSTEM_CRMD;
- crmd_operation = CRM_OP_VOTE;
- ret = 0; /* no return message */
-
- } else if (DO_WHOIS_DC) {
- dest_node = NULL;
- sys_to = CRM_SYSTEM_DC;
- crmd_operation = CRM_OP_PING;
-
- } else if (DO_NODE_LIST) {
- crm_exit(pcmk_rc2exitc(list_nodes()));
-
- } else if (DO_RESET) {
- /* tell dest_node to initiate the shutdown procedure
- *
- * if dest_node is NULL, the request will be sent to the
- * local node
- */
- sys_to = CRM_SYSTEM_CRMD;
- ret = 0; /* no return message */
-
- } else {
- crm_err("Unknown options");
- all_is_good = FALSE;
}
- if (all_is_good == FALSE) {
- crm_err("Creation of request failed. No message to send");
- return -1;
+ if (do_work(controld_api)) {
+ // A reply is needed from controller, so run main loop to get it
+ exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
+ mainloop = g_main_loop_new(NULL, FALSE);
+ message_timer_id = g_timeout_add(message_timeout_ms,
+ admin_message_timeout, NULL);
+ g_main_loop_run(mainloop);
}
-/* send it */
- if (crmd_channel == NULL) {
- crm_err("The IPC connection is not valid, cannot send anything");
- return -1;
- }
-
- if (sys_to == NULL) {
- if (dest_node != NULL) {
- sys_to = CRM_SYSTEM_CRMD;
- } else {
- sys_to = CRM_SYSTEM_DC;
- }
- }
-
- {
- xmlNode *cmd = create_request(crmd_operation, msg_data, dest_node, sys_to,
- crm_system_name, admin_uuid);
-
- crm_ipc_send(crmd_channel, cmd, 0, 0, NULL);
- free_xml(cmd);
- }
-
- return ret;
-}
+done:
+ if (controld_api != NULL) {
+ pcmk_ipc_api_t *capi = controld_api;
-void
-crmadmin_ipc_connection_destroy(gpointer user_data)
-{
- crm_err("Connection to controller was terminated");
- if (mainloop) {
- g_main_loop_quit(mainloop);
- } else {
- crm_exit(CRM_EX_DISCONNECT);
+ controld_api = NULL; // Ensure we can't free this twice
+ pcmk_free_ipc_api(capi);
}
-}
-
-struct ipc_client_callbacks crm_callbacks = {
- .dispatch = admin_msg_callback,
- .destroy = crmadmin_ipc_connection_destroy
-};
-
-gboolean
-do_init(void)
-{
- mainloop_io_t *source =
- mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks);
-
- admin_uuid = pcmk__getpid_s();
-
- crmd_channel = mainloop_get_ipc_client(source);
-
- if (DO_RESOURCE || DO_RESOURCE_LIST || DO_NODE_LIST) {
- return TRUE;
-
- } else if (crmd_channel != NULL) {
- xmlNode *xml = create_hello_message(admin_uuid, crm_system_name, "0", "1");
-
- crm_ipc_send(crmd_channel, xml, 0, 0, NULL);
- return TRUE;
+ if (mainloop != NULL) {
+ g_main_loop_unref(mainloop);
+ mainloop = NULL;
}
- return FALSE;
+ return crm_exit(exit_code);
}
-static bool
-validate_crm_message(xmlNode * msg, const char *sys, const char *uuid, const char *msg_type)
+// \return True if reply from controller is needed
+bool
+do_work(pcmk_ipc_api_t *controld_api)
{
- const char *type = NULL;
- const char *crm_msg_reference = NULL;
-
- if (msg == NULL) {
- return FALSE;
- }
+ bool need_reply = false;
+ int rc = pcmk_rc_ok;
- type = crm_element_value(msg, F_CRM_MSG_TYPE);
- crm_msg_reference = crm_element_value(msg, XML_ATTR_REFERENCE);
-
- if (type == NULL) {
- crm_info("No message type defined.");
- return FALSE;
-
- } else if (msg_type != NULL && strcasecmp(msg_type, type) != 0) {
- crm_info("Expecting a (%s) message but received a (%s).", msg_type, type);
- return FALSE;
- }
-
- if (crm_msg_reference == NULL) {
- crm_info("No message crm_msg_reference defined.");
- return FALSE;
- }
-
- return TRUE;
-}
-
-int
-admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata)
-{
- static int received_responses = 0;
- xmlNode *xml = string2xml(buffer);
-
- received_responses++;
- g_source_remove(message_timer_id);
-
- crm_log_xml_trace(xml, "ipc");
-
- if (xml == NULL) {
- crm_info("XML in IPC message was not valid... " "discarding.");
-
- } else if (validate_crm_message(xml, crm_system_name, admin_uuid, XML_ATTR_RESPONSE) == FALSE) {
- crm_trace("Message was not a CRM response. Discarding.");
-
- } else if (DO_HEALTH) {
- xmlNode *data = get_message_xml(xml, F_CRM_DATA);
- const char *state = crm_element_value(data, XML_PING_ATTR_CRMDSTATE);
+ switch (command) {
+ case cmd_shutdown:
+ rc = pcmk_controld_api_shutdown(controld_api, dest_node);
+ break;
- printf("Status of %s@%s: %s (%s)\n",
- crm_element_value(data, XML_PING_ATTR_SYSFROM),
- crm_element_value(xml, F_CRM_HOST_FROM),
- state, crm_element_value(data, XML_PING_ATTR_STATUS));
+ case cmd_health: // dest_node != NULL
+ case cmd_whois_dc: // dest_node == NULL
+ rc = pcmk_controld_api_ping(controld_api, dest_node);
+ need_reply = true;
+ break;
- if (BE_SILENT && state != NULL) {
- fprintf(stderr, "%s\n", state);
- }
+ case cmd_elect_dc:
+ rc = pcmk_controld_api_start_election(controld_api);
+ break;
- } else if (DO_WHOIS_DC) {
- const char *dc = crm_element_value(xml, F_CRM_HOST_FROM);
+ case cmd_list_nodes:
+ rc = list_nodes();
+ break;
- printf("Designated Controller is: %s\n", dc);
- if (BE_SILENT && dc != NULL) {
- fprintf(stderr, "%s\n", dc);
- }
- crm_exit(CRM_EX_OK);
+ case cmd_none: // not actually possible here
+ break;
}
-
- free_xml(xml);
-
- if (received_responses >= expected_responses) {
- crm_trace("Received expected number (%d) of replies, exiting normally",
- expected_responses);
- crm_exit(CRM_EX_OK);
+ if (rc != pcmk_rc_ok) {
+ fprintf(stderr, "error: Command failed: %s", pcmk_rc_str(rc));
+ exit_code = pcmk_rc2exitc(rc);
}
-
- message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL);
- return 0;
+ return need_reply;
}
gboolean
admin_message_timeout(gpointer data)
{
- fprintf(stderr, "No messages received in %d seconds.. aborting\n",
- (int)message_timeout_ms / 1000);
- crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000);
- exit_code = CRM_EX_TIMEOUT;
- g_main_loop_quit(mainloop);
- return FALSE;
+ fprintf(stderr,
+ "error: No reply received from controller before timeout (%dms)\n",
+ message_timeout_ms);
+ message_timer_id = 0;
+ quit_main_loop(CRM_EX_TIMEOUT);
+ return FALSE; // Tells glib to remove source
}
-int
+void
do_find_node_list(xmlNode * xml_node)
{
int found = 0;
xmlNode *node = NULL;
xmlNode *nodes = get_object_root(XML_CIB_TAG_NODES, xml_node);
- for (node = __xml_first_child_element(nodes); node != NULL;
- node = __xml_next_element(node)) {
+ for (node = first_named_child(nodes, XML_CIB_TAG_NODE); node != NULL;
+ node = crm_next_same_xml(node)) {
- if (crm_str_eq((const char *)node->name, XML_CIB_TAG_NODE, TRUE)) {
+ if (BASH_EXPORT) {
+ printf("export %s=%s\n",
+ crm_element_value(node, XML_ATTR_UNAME),
+ crm_element_value(node, XML_ATTR_ID));
+ } else {
+ const char *node_type = crm_element_value(node, XML_ATTR_TYPE);
- if (BASH_EXPORT) {
- printf("export %s=%s\n",
- crm_element_value(node, XML_ATTR_UNAME),
- crm_element_value(node, XML_ATTR_ID));
- } else {
- printf("%s node: %s (%s)\n",
- crm_element_value(node, XML_ATTR_TYPE),
- crm_element_value(node, XML_ATTR_UNAME),
- crm_element_value(node, XML_ATTR_ID));
+ if (node_type == NULL) {
+ node_type = "member";
}
- found++;
+ printf("%s node: %s (%s)\n", node_type,
+ crm_element_value(node, XML_ATTR_UNAME),
+ crm_element_value(node, XML_ATTR_ID));
}
+ found++;
}
+ // @TODO List Pacemaker Remote nodes that don't have a <node> entry
if (found == 0) {
- printf("NO nodes configured\n");
+ printf("No nodes configured\n");
}
-
- return found;
}
--
1.8.3.1