From e34421b2608f235c6a77eb6de4596d93a2128be1 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Fri, 3 Apr 2020 11:13:52 -0500 Subject: [PATCH 1/6] Refactor: libcrmcommon: new model for daemon IPC API This is based on tools/crm_resource_controller.[ch] and the node removal code in tools/crm_node.c, but generalized for any daemon. As of this commit, no specific daemon is supported. --- include/crm/common/internal.h | 8 + include/crm/common/ipc.h | 95 +++++- lib/common/crmcommon_private.h | 86 +++++ lib/common/ipc_client.c | 703 ++++++++++++++++++++++++++++++++++++++++- lib/common/mainloop.c | 70 ++-- 5 files changed, 940 insertions(+), 22 deletions(-) diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h index 28b20b4..13c29ea 100644 --- a/include/crm/common/internal.h +++ b/include/crm/common/internal.h @@ -19,6 +19,7 @@ #include // xmlNode #include // crm_strdup_printf() +#include // mainloop_io_t, struct ipc_client_callbacks // Internal ACL-related utilities (from acl.c) @@ -103,6 +104,13 @@ pcmk__open_devnull(int flags) } while (0) +/* internal main loop utilities (from mainloop.c) */ + +int pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata, + struct ipc_client_callbacks *callbacks, + mainloop_io_t **source); + + /* internal procfs utilities (from procfs.c) */ pid_t pcmk__procfs_pid_of(const char *name); diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h index a0df956..8dee1b1 100644 --- a/include/crm/common/ipc.h +++ b/include/crm/common/ipc.h @@ -16,7 +16,8 @@ extern "C" { /** * \file - * \brief Wrappers for and extensions to libqb IPC + * \brief IPC interface to Pacemaker daemons + * * \ingroup core */ @@ -48,6 +49,96 @@ xmlNode *create_request_adv(const char *task, xmlNode *xml_data, const char *origin); +/* + * The library supports two methods of creating IPC connections. The older code + * allows connecting to any arbitrary IPC name. The newer code only allows + * connecting to one of the Pacemaker daemons. + * + * As daemons are converted to use the new model, the old functions should be + * considered deprecated for use with those daemons. Once all daemons are + * converted, the old functions should be officially deprecated as public API + * and eventually made internal API. + */ + +/* + * Pacemaker daemon IPC + */ + +//! Available IPC interfaces +enum pcmk_ipc_server { + pcmk_ipc_attrd, //!< Attribute manager + pcmk_ipc_based, //!< CIB manager + pcmk_ipc_controld, //!< Controller + pcmk_ipc_execd, //!< Executor + pcmk_ipc_fenced, //!< Fencer + pcmk_ipc_pacemakerd, //!< Launcher + pcmk_ipc_schedulerd, //!< Scheduler +}; + +//! Possible event types that an IPC event callback can be called for +enum pcmk_ipc_event { + pcmk_ipc_event_connect, //!< Result of asynchronous connection attempt + pcmk_ipc_event_disconnect, //!< Termination of IPC connection + pcmk_ipc_event_reply, //!< Daemon's reply to client IPC request + pcmk_ipc_event_notify, //!< Notification from daemon +}; + +//! How IPC replies should be dispatched +enum pcmk_ipc_dispatch { + pcmk_ipc_dispatch_main, //!< Attach IPC to GMainLoop for dispatch + pcmk_ipc_dispatch_poll, //!< Caller will poll and dispatch IPC + pcmk_ipc_dispatch_sync, //!< Sending a command will wait for any reply +}; + +//! Client connection to Pacemaker IPC +typedef struct pcmk_ipc_api_s pcmk_ipc_api_t; + +/*! + * \brief Callback function type for Pacemaker daemon IPC APIs + * + * \param[in] api IPC API connection + * \param[in] event_type The type of event that occurred + * \param[in] status Event status + * \param[in] event_data Event-specific data + * \param[in] user_data Caller data provided when callback was registered + * + * \note For connection and disconnection events, event_data may be NULL (for + * local IPC) or the name of the connected node (for remote IPC, for + * daemons that support that). For reply and notify events, event_data is + * defined by the specific daemon API. + */ +typedef void (*pcmk_ipc_callback_t)(pcmk_ipc_api_t *api, + enum pcmk_ipc_event event_type, + crm_exit_t status, + void *event_data, void *user_data); + +int pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server); + +void pcmk_free_ipc_api(pcmk_ipc_api_t *api); + +int pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type); + +void pcmk_disconnect_ipc(pcmk_ipc_api_t *api); + +int pcmk_poll_ipc(pcmk_ipc_api_t *api, int timeout_ms); + +void pcmk_dispatch_ipc(pcmk_ipc_api_t *api); + +void pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb, + void *user_data); + +const char *pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log); + +bool pcmk_ipc_is_connected(pcmk_ipc_api_t *api); + +int pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name, + uint32_t nodeid); + + +/* + * Generic IPC API (to eventually be deprecated as public API and made internal) + */ + /* *INDENT-OFF* */ enum crm_ipc_flags { @@ -58,7 +149,7 @@ enum crm_ipc_flags crm_ipc_proxied = 0x00000100, /* _ALL_ replies to proxied connections need to be sent as events */ crm_ipc_client_response = 0x00000200, /* A Response is expected in reply */ - // These are options only for pcmk__ipc_send_iov() + // These are options for Pacemaker's internal use only (pcmk__ipc_send_*()) crm_ipc_server_event = 0x00010000, /* Send an Event instead of a Response */ crm_ipc_server_free = 0x00020000, /* Free the iovec after sending */ crm_ipc_proxied_relay_response = 0x00040000, /* all replies to proxied connections are sent as events, this flag preserves whether the event should be treated as an actual event, or a response.*/ diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h index d06fa20..f9df27d 100644 --- a/lib/common/crmcommon_private.h +++ b/lib/common/crmcommon_private.h @@ -103,6 +103,84 @@ pcmk__xml_attr_value(const xmlAttr *attr) #define PCMK__IPC_VERSION 1 +// IPC behavior that varies by daemon +typedef struct pcmk__ipc_methods_s { + /*! + * \internal + * \brief Allocate any private data needed by daemon IPC + * + * \param[in] api IPC API connection + * + * \return Standard Pacemaker return code + */ + int (*new_data)(pcmk_ipc_api_t *api); + + /*! + * \internal + * \brief Free any private data used by daemon IPC + * + * \param[in] api_data Data allocated by new_data() method + */ + void (*free_data)(void *api_data); + + /*! + * \internal + * \brief Perform daemon-specific handling after successful connection + * + * Some daemons require clients to register before sending any other + * commands. The controller requires a CRM_OP_HELLO (with no reply), and + * the CIB manager, executor, and fencer require a CRM_OP_REGISTER (with a + * reply). Ideally this would be consistent across all daemons, but for now + * this allows each to do its own authorization. + * + * \param[in] api IPC API connection + * + * \return Standard Pacemaker return code + */ + int (*post_connect)(pcmk_ipc_api_t *api); + + /*! + * \internal + * \brief Check whether an IPC request results in a reply + * + * \parma[in] api IPC API connection + * \param[in] request IPC request XML + * + * \return true if request would result in an IPC reply, false otherwise + */ + bool (*reply_expected)(pcmk_ipc_api_t *api, xmlNode *request); + + /*! + * \internal + * \brief Perform daemon-specific handling of an IPC message + * + * \param[in] api IPC API connection + * \param[in] msg Message read from IPC connection + */ + void (*dispatch)(pcmk_ipc_api_t *api, xmlNode *msg); + + /*! + * \internal + * \brief Perform daemon-specific handling of an IPC disconnect + * + * \param[in] api IPC API connection + */ + void (*post_disconnect)(pcmk_ipc_api_t *api); +} pcmk__ipc_methods_t; + +// Implementation of pcmk_ipc_api_t +struct pcmk_ipc_api_s { + enum pcmk_ipc_server server; // Daemon this IPC API instance is for + enum pcmk_ipc_dispatch dispatch_type; // How replies should be dispatched + crm_ipc_t *ipc; // IPC connection + mainloop_io_t *mainloop_io; // If using mainloop, I/O source for IPC + bool free_on_disconnect; // Whether disconnect should free object + pcmk_ipc_callback_t cb; // Caller-registered callback (if any) + void *user_data; // Caller-registered data (if any) + void *api_data; // For daemon-specific use + pcmk__ipc_methods_t *cmds; // Behavior that varies by daemon +}; + typedef struct pcmk__ipc_header_s { struct qb_ipc_response_header qb; uint32_t size_uncompressed; @@ -112,6 +190,14 @@ typedef struct pcmk__ipc_header_s { } pcmk__ipc_header_t; G_GNUC_INTERNAL +int pcmk__send_ipc_request(pcmk_ipc_api_t *api, xmlNode *request); + +G_GNUC_INTERNAL +void pcmk__call_ipc_callback(pcmk_ipc_api_t *api, + enum pcmk_ipc_event event_type, + crm_exit_t status, void *event_data); + +G_GNUC_INTERNAL unsigned int pcmk__ipc_buffer_size(unsigned int max); G_GNUC_INTERNAL diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c index 7737588..16dc9b5 100644 --- a/lib/common/ipc_client.c +++ b/lib/common/ipc_client.c @@ -31,6 +31,679 @@ #include #include "crmcommon_private.h" +/*! + * \brief Create a new object for using Pacemaker daemon IPC + * + * \param[out] api Where to store new IPC object + * \param[in] server Which Pacemaker daemon the object is for + * + * \return Standard Pacemaker result code + * + * \note The caller is responsible for freeing *api using pcmk_free_ipc_api(). + * \note This is intended to supersede crm_ipc_new() but is not yet usable. + */ +int +pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server) +{ + size_t max_size = 0; + + if (api == NULL) { + return EINVAL; + } + + *api = calloc(1, sizeof(pcmk_ipc_api_t)); + if (*api == NULL) { + return errno; + } + + (*api)->server = server; + if (pcmk_ipc_name(*api, false) == NULL) { + pcmk_free_ipc_api(*api); + *api = NULL; + return EOPNOTSUPP; + } + + // Set server methods and max_size (if not default) + switch (server) { + case pcmk_ipc_attrd: + break; + + case pcmk_ipc_based: + max_size = 512 * 1024; // 512KB + break; + + case pcmk_ipc_controld: + break; + + case pcmk_ipc_execd: + break; + + case pcmk_ipc_fenced: + break; + + case pcmk_ipc_pacemakerd: + break; + + case pcmk_ipc_schedulerd: + // @TODO max_size could vary by client, maybe take as argument? + max_size = 5 * 1024 * 1024; // 5MB + break; + } + if ((*api)->cmds == NULL) { + pcmk_free_ipc_api(*api); + *api = NULL; + return ENOMEM; + } + + (*api)->ipc = crm_ipc_new(pcmk_ipc_name(*api, false), max_size); + if ((*api)->ipc == NULL) { + pcmk_free_ipc_api(*api); + *api = NULL; + return ENOMEM; + } + + // If daemon API has its own data to track, allocate it + if ((*api)->cmds->new_data != NULL) { + if ((*api)->cmds->new_data(*api) != pcmk_rc_ok) { + pcmk_free_ipc_api(*api); + *api = NULL; + return ENOMEM; + } + } + crm_trace("Created %s API IPC object", pcmk_ipc_name(*api, true)); + return pcmk_rc_ok; +} + +static void +free_daemon_specific_data(pcmk_ipc_api_t *api) +{ + if ((api != NULL) && (api->cmds != NULL)) { + if ((api->cmds->free_data != NULL) && (api->api_data != NULL)) { + api->cmds->free_data(api->api_data); + api->api_data = NULL; + } + free(api->cmds); + api->cmds = NULL; + } +} + +/*! + * \internal + * \brief Call an IPC API event callback, if one is registed + * + * \param[in] api IPC API connection + * \param[in] event_type The type of event that occurred + * \param[in] status Event status + * \param[in] event_data Event-specific data + */ +void +pcmk__call_ipc_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type, + crm_exit_t status, void *event_data) +{ + if ((api != NULL) && (api->cb != NULL)) { + api->cb(api, event_type, status, event_data, api->user_data); + } +} + +/*! + * \internal + * \brief Clean up after an IPC disconnect + * + * \param[in] user_data IPC API connection that disconnected + * + * \note This function can be used as a main loop IPC destroy callback. + */ +static void +ipc_post_disconnect(gpointer user_data) +{ + pcmk_ipc_api_t *api = user_data; + + crm_info("Disconnected from %s IPC API", pcmk_ipc_name(api, true)); + + // Perform any daemon-specific handling needed + if ((api->cmds != NULL) && (api->cmds->post_disconnect != NULL)) { + api->cmds->post_disconnect(api); + } + + // Call client's registered event callback + pcmk__call_ipc_callback(api, pcmk_ipc_event_disconnect, CRM_EX_DISCONNECT, + NULL); + + /* If this is being called from a running main loop, mainloop_gio_destroy() + * will free ipc and mainloop_io immediately after calling this function. + * If this is called from a stopped main loop, these will leak, so the best + * practice is to close the connection before stopping the main loop. + */ + api->ipc = NULL; + api->mainloop_io = NULL; + + if (api->free_on_disconnect) { + /* pcmk_free_ipc_api() has already been called, but did not free api + * or api->cmds because this function needed them. Do that now. + */ + free_daemon_specific_data(api); + crm_trace("Freeing IPC API object after disconnect"); + free(api); + } +} + +/*! + * \brief Free the contents of an IPC API object + * + * \param[in] api IPC API object to free + */ +void +pcmk_free_ipc_api(pcmk_ipc_api_t *api) +{ + bool free_on_disconnect = false; + + if (api == NULL) { + return; + } + crm_debug("Releasing %s IPC API", pcmk_ipc_name(api, true)); + + if (api->ipc != NULL) { + if (api->mainloop_io != NULL) { + /* We need to keep the api pointer itself around, because it is the + * user data for the IPC client destroy callback. That will be + * triggered by the pcmk_disconnect_ipc() call below, but it might + * happen later in the main loop (if still running). + * + * This flag tells the destroy callback to free the object. It can't + * do that unconditionally, because the application might call this + * function after a disconnect that happened by other means. + */ + free_on_disconnect = api->free_on_disconnect = true; + } + pcmk_disconnect_ipc(api); // Frees api if free_on_disconnect is true + } + if (!free_on_disconnect) { + free_daemon_specific_data(api); + crm_trace("Freeing IPC API object"); + free(api); + } +} + +/*! + * \brief Get the IPC name used with an IPC API connection + * + * \param[in] api IPC API connection + * \param[in] for_log If true, return human-friendly name instead of IPC name + * + * \return IPC API's human-friendly or connection name, or if none is available, + * "Pacemaker" if for_log is true and NULL if for_log is false + */ +const char * +pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log) +{ + if (api == NULL) { + return for_log? "Pacemaker" : NULL; + } + switch (api->server) { + case pcmk_ipc_attrd: + return for_log? "attribute manager" : NULL /* T_ATTRD */; + + case pcmk_ipc_based: + return for_log? "CIB manager" : NULL /* PCMK__SERVER_BASED_RW */; + + case pcmk_ipc_controld: + return for_log? "controller" : NULL /* CRM_SYSTEM_CRMD */; + + case pcmk_ipc_execd: + return for_log? "executor" : NULL /* CRM_SYSTEM_LRMD */; + + case pcmk_ipc_fenced: + return for_log? "fencer" : NULL /* "stonith-ng" */; + + case pcmk_ipc_pacemakerd: + return for_log? "launcher" : NULL /* CRM_SYSTEM_MCP */; + + case pcmk_ipc_schedulerd: + return for_log? "scheduler" : NULL /* CRM_SYSTEM_PENGINE */; + + default: + return for_log? "Pacemaker" : NULL; + } +} + +/*! + * \brief Check whether an IPC API connection is active + * + * \param[in] api IPC API connection + * + * \return true if IPC is connected, false otherwise + */ +bool +pcmk_ipc_is_connected(pcmk_ipc_api_t *api) +{ + return (api != NULL) && crm_ipc_connected(api->ipc); +} + +/*! + * \internal + * \brief Call the daemon-specific API's dispatch function + * + * Perform daemon-specific handling of IPC reply dispatch. It is the daemon + * method's responsibility to call the client's registered event callback, as + * well as allocate and free any event data. + * + * \param[in] api IPC API connection + */ +static void +call_api_dispatch(pcmk_ipc_api_t *api, xmlNode *message) +{ + crm_log_xml_trace(message, "ipc-received"); + if ((api->cmds != NULL) && (api->cmds->dispatch != NULL)) { + api->cmds->dispatch(api, message); + } +} + +/*! + * \internal + * \brief Dispatch data read from IPC source + * + * \param[in] buffer Data read from IPC + * \param[in] length Number of bytes of data in buffer (ignored) + * \param[in] user_data IPC object + * + * \return Always 0 (meaning connection is still required) + * + * \note This function can be used as a main loop IPC dispatch callback. + */ +static int +dispatch_ipc_data(const char *buffer, ssize_t length, gpointer user_data) +{ + pcmk_ipc_api_t *api = user_data; + xmlNode *msg; + + CRM_CHECK(api != NULL, return 0); + + if (buffer == NULL) { + crm_warn("Empty message received from %s IPC", + pcmk_ipc_name(api, true)); + return 0; + } + + msg = string2xml(buffer); + if (msg == NULL) { + crm_warn("Malformed message received from %s IPC", + pcmk_ipc_name(api, true)); + return 0; + } + call_api_dispatch(api, msg); + free_xml(msg); + return 0; +} + +/*! + * \brief Check whether an IPC connection has data available (without main loop) + * + * \param[in] api IPC API connection + * \param[in] timeout_ms If less than 0, poll indefinitely; if 0, poll once + * and return immediately; otherwise, poll for up to + * this many milliseconds + * + * \return Standard Pacemaker return code + * + * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call + * this function to check whether IPC data is available. Return values of + * interest include pcmk_rc_ok meaning data is available, and EAGAIN + * meaning no data is available; all other values indicate errors. + * \todo This does not allow the caller to poll multiple file descriptors at + * once. If there is demand for that, we could add a wrapper for + * crm_ipc_get_fd(api->ipc), so the caller can call poll() themselves. + */ +int +pcmk_poll_ipc(pcmk_ipc_api_t *api, int timeout_ms) +{ + int rc; + struct pollfd pollfd = { 0, }; + + if ((api == NULL) || (api->dispatch_type != pcmk_ipc_dispatch_poll)) { + return EINVAL; + } + pollfd.fd = crm_ipc_get_fd(api->ipc); + pollfd.events = POLLIN; + rc = poll(&pollfd, 1, timeout_ms); + if (rc < 0) { + return errno; + } else if (rc == 0) { + return EAGAIN; + } + return pcmk_rc_ok; +} + +/*! + * \brief Dispatch available messages on an IPC connection (without main loop) + * + * \param[in] api IPC API connection + * + * \return Standard Pacemaker return code + * + * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call + * this function when IPC data is available. + */ +void +pcmk_dispatch_ipc(pcmk_ipc_api_t *api) +{ + if (api == NULL) { + return; + } + while (crm_ipc_ready(api->ipc)) { + if (crm_ipc_read(api->ipc) > 0) { + dispatch_ipc_data(crm_ipc_buffer(api->ipc), 0, api); + } + } +} + +// \return Standard Pacemaker return code +static int +connect_with_main_loop(pcmk_ipc_api_t *api) +{ + int rc; + + struct ipc_client_callbacks callbacks = { + .dispatch = dispatch_ipc_data, + .destroy = ipc_post_disconnect, + }; + + rc = pcmk__add_mainloop_ipc(api->ipc, G_PRIORITY_DEFAULT, api, + &callbacks, &(api->mainloop_io)); + if (rc != pcmk_rc_ok) { + return rc; + } + crm_debug("Connected to %s IPC (attached to main loop)", + pcmk_ipc_name(api, true)); + /* After this point, api->mainloop_io owns api->ipc, so api->ipc + * should not be explicitly freed. + */ + return pcmk_rc_ok; +} + +// \return Standard Pacemaker return code +static int +connect_without_main_loop(pcmk_ipc_api_t *api) +{ + int rc; + + if (!crm_ipc_connect(api->ipc)) { + rc = errno; + crm_ipc_close(api->ipc); + return rc; + } + crm_debug("Connected to %s IPC (without main loop)", + pcmk_ipc_name(api, true)); + return pcmk_rc_ok; +} + +/*! + * \brief Connect to a Pacemaker daemon via IPC + * + * \param[in] api IPC API instance + * \param[out] dispatch_type How IPC replies should be dispatched + * + * \return Standard Pacemaker return code + */ +int +pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type) +{ + int rc = pcmk_rc_ok; + + if ((api == NULL) || (api->ipc == NULL)) { + crm_err("Cannot connect to uninitialized API object"); + return EINVAL; + } + + if (crm_ipc_connected(api->ipc)) { + crm_trace("Already connected to %s IPC API", pcmk_ipc_name(api, true)); + return pcmk_rc_ok; + } + + api->dispatch_type = dispatch_type; + switch (dispatch_type) { + case pcmk_ipc_dispatch_main: + rc = connect_with_main_loop(api); + break; + + case pcmk_ipc_dispatch_sync: + case pcmk_ipc_dispatch_poll: + rc = connect_without_main_loop(api); + break; + } + if (rc != pcmk_rc_ok) { + return rc; + } + + if ((api->cmds != NULL) && (api->cmds->post_connect != NULL)) { + rc = api->cmds->post_connect(api); + if (rc != pcmk_rc_ok) { + crm_ipc_close(api->ipc); + } + } + return rc; +} + +/*! + * \brief Disconnect an IPC API instance + * + * \param[in] api IPC API connection + * + * \return Standard Pacemaker return code + * + * \note If the connection is attached to a main loop, this function should be + * called before quitting the main loop, to ensure that all memory is + * freed. + */ +void +pcmk_disconnect_ipc(pcmk_ipc_api_t *api) +{ + if ((api == NULL) || (api->ipc == NULL)) { + return; + } + switch (api->dispatch_type) { + case pcmk_ipc_dispatch_main: + { + mainloop_io_t *mainloop_io = api->mainloop_io; + + // Make sure no code with access to api can use these again + api->mainloop_io = NULL; + api->ipc = NULL; + + mainloop_del_ipc_client(mainloop_io); + // After this point api might have already been freed + } + break; + + case pcmk_ipc_dispatch_poll: + case pcmk_ipc_dispatch_sync: + { + crm_ipc_t *ipc = api->ipc; + + // Make sure no code with access to api can use ipc again + api->ipc = NULL; + + // This should always be the case already, but to be safe + api->free_on_disconnect = false; + + crm_ipc_destroy(ipc); + ipc_post_disconnect(api); + } + break; + } +} + +/*! + * \brief Register a callback for IPC API events + * + * \param[in] api IPC API connection + * \param[in] callback Callback to register + * \param[in] userdata Caller data to pass to callback + * + * \note This function may be called multiple times to update the callback + * and/or user data. The caller remains responsible for freeing + * userdata in any case (after the IPC is disconnected, if the + * user data is still registered with the IPC). + */ +void +pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb, + void *user_data) +{ + if (api == NULL) { + return; + } + api->cb = cb; + api->user_data = user_data; +} + +/*! + * \internal + * \brief Send an XML request across an IPC API connection + * + * \param[in] api IPC API connection + * \param[in] request XML request to send + * + * \return Standard Pacemaker return code + * + * \note Daemon-specific IPC API functions should call this function to send + * requests, because it handles different dispatch types appropriately. + */ +int +pcmk__send_ipc_request(pcmk_ipc_api_t *api, xmlNode *request) +{ + int rc; + xmlNode *reply = NULL; + enum crm_ipc_flags flags = crm_ipc_flags_none; + + if ((api == NULL) || (api->ipc == NULL) || (request == NULL)) { + return EINVAL; + } + crm_log_xml_trace(request, "ipc-sent"); + + // Synchronous dispatch requires waiting for a reply + if ((api->dispatch_type == pcmk_ipc_dispatch_sync) + && (api->cmds != NULL) + && (api->cmds->reply_expected != NULL) + && (api->cmds->reply_expected(api, request))) { + flags = crm_ipc_client_response; + } + + // The 0 here means a default timeout of 5 seconds + rc = crm_ipc_send(api->ipc, request, flags, 0, &reply); + + if (rc < 0) { + return pcmk_legacy2rc(rc); + } else if (rc == 0) { + return ENODATA; + } + + // With synchronous dispatch, we dispatch any reply now + if (reply != NULL) { + call_api_dispatch(api, reply); + free_xml(reply); + } + return pcmk_rc_ok; +} + +/*! + * \internal + * \brief Create the XML for an IPC request to purge a node from the peer cache + * + * \param[in] api IPC API connection + * \param[in] node_name If not NULL, name of node to purge + * \param[in] nodeid If not 0, node ID of node to purge + * + * \return Newly allocated IPC request XML + * + * \note The controller, fencer, and pacemakerd use the same request syntax, but + * the attribute manager uses a different one. The CIB manager doesn't + * have any syntax for it. The executor and scheduler don't connect to the + * cluster layer and thus don't have or need any syntax for it. + * + * \todo Modify the attribute manager to accept the common syntax (as well + * as its current one, for compatibility with older clients). Modify + * the CIB manager to accept and honor the common syntax. Modify the + * executor and scheduler to accept the syntax (immediately returning + * success), just for consistency. Modify this function to use the + * common syntax with all daemons if their version supports it. + */ +static xmlNode * +create_purge_node_request(pcmk_ipc_api_t *api, const char *node_name, + uint32_t nodeid) +{ + xmlNode *request = NULL; + const char *client = crm_system_name? crm_system_name : "client"; + + switch (api->server) { + case pcmk_ipc_attrd: + request = create_xml_node(NULL, __FUNCTION__); + crm_xml_add(request, F_TYPE, T_ATTRD); + crm_xml_add(request, F_ORIG, crm_system_name); + crm_xml_add(request, PCMK__XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE); + crm_xml_add(request, PCMK__XA_ATTR_NODE_NAME, node_name); + if (nodeid > 0) { + crm_xml_add_int(request, PCMK__XA_ATTR_NODE_ID, (int) nodeid); + } + break; + + case pcmk_ipc_controld: + case pcmk_ipc_fenced: + case pcmk_ipc_pacemakerd: + request = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, + pcmk_ipc_name(api, false), client, NULL); + if (nodeid > 0) { + crm_xml_set_id(request, "%lu", (unsigned long) nodeid); + } + crm_xml_add(request, XML_ATTR_UNAME, node_name); + break; + + case pcmk_ipc_based: + case pcmk_ipc_execd: + case pcmk_ipc_schedulerd: + break; + } + return request; +} + +/*! + * \brief Ask a Pacemaker daemon to purge a node from its peer cache + * + * \param[in] api IPC API connection + * \param[in] node_name If not NULL, name of node to purge + * \param[in] nodeid If not 0, node ID of node to purge + * + * \return Standard Pacemaker return code + * + * \note At least one of node_name or nodeid must be specified. + */ +int +pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name, uint32_t nodeid) +{ + int rc = 0; + xmlNode *request = NULL; + + if (api == NULL) { + return EINVAL; + } + if ((node_name == NULL) && (nodeid == 0)) { + return EINVAL; + } + + request = create_purge_node_request(api, node_name, nodeid); + if (request == NULL) { + return EOPNOTSUPP; + } + rc = pcmk__send_ipc_request(api, request); + free_xml(request); + + crm_debug("%s peer cache purge of node %s[%lu]: rc=%d", + pcmk_ipc_name(api, true), node_name, (unsigned long) nodeid, rc); + return rc; +} + +/* + * Generic IPC API (to eventually be deprecated as public API and made internal) + */ + struct crm_ipc_s { struct pollfd pfd; unsigned int max_buf_size; // maximum bytes we can send or receive over IPC @@ -42,16 +715,44 @@ struct crm_ipc_s { qb_ipcc_connection_t *ipc; }; +/*! + * \brief Create a new (legacy) object for using Pacemaker daemon IPC + * + * \param[in] name IPC system name to connect to + * \param[in] max_size Use a maximum IPC buffer size of at least this size + * + * \return Newly allocated IPC object on success, NULL otherwise + * + * \note The caller is responsible for freeing the result using + * crm_ipc_destroy(). + * \note This should be considered deprecated for use with daemons supported by + * pcmk_new_ipc_api(). + */ crm_ipc_t * crm_ipc_new(const char *name, size_t max_size) { crm_ipc_t *client = NULL; client = calloc(1, sizeof(crm_ipc_t)); + if (client == NULL) { + crm_err("Could not create IPC connection: %s", strerror(errno)); + return NULL; + } client->name = strdup(name); + if (client->name == NULL) { + crm_err("Could not create IPC connection: %s", strerror(errno)); + free(client); + return NULL; + } client->buf_size = pcmk__ipc_buffer_size(max_size); client->buffer = malloc(client->buf_size); + if (client->buffer == NULL) { + crm_err("Could not create IPC connection: %s", strerror(errno)); + free(client->name); + free(client); + return NULL; + } /* Clients initiating connection pick the max buf size */ client->max_buf_size = client->buf_size; @@ -143,8 +844,6 @@ void crm_ipc_close(crm_ipc_t * client) { if (client) { - crm_trace("Disconnecting %s IPC connection %p (%p)", client->name, client, client->ipc); - if (client->ipc) { qb_ipcc_connection_t *ipc = client->ipc; diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c index 634eead..e942e57 100644 --- a/lib/common/mainloop.c +++ b/lib/common/mainloop.c @@ -834,32 +834,66 @@ mainloop_gio_destroy(gpointer c) free(c_name); } -mainloop_io_t * -mainloop_add_ipc_client(const char *name, int priority, size_t max_size, void *userdata, - struct ipc_client_callbacks *callbacks) +/*! + * \brief Connect to IPC and add it as a main loop source + * + * \param[in] ipc IPC connection to add + * \param[in] priority Event source priority to use for connection + * \param[in] userdata Data to register with callbacks + * \param[in] callbacks Dispatch and destroy callbacks for connection + * \param[out] source Newly allocated event source + * + * \return Standard Pacemaker return code + * + * \note On failure, the caller is still responsible for ipc. On success, the + * caller should call mainloop_del_ipc_client() when source is no longer + * needed, which will lead to the disconnection of the IPC later in the + * main loop if it is connected. However the IPC disconnects, + * mainloop_gio_destroy() will free ipc and source after calling the + * destroy callback. + */ +int +pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata, + struct ipc_client_callbacks *callbacks, + mainloop_io_t **source) { - mainloop_io_t *client = NULL; - crm_ipc_t *conn = crm_ipc_new(name, max_size); + CRM_CHECK((ipc != NULL) && (callbacks != NULL), return EINVAL); - if (conn && crm_ipc_connect(conn)) { - int32_t fd = crm_ipc_get_fd(conn); + if (!crm_ipc_connect(ipc)) { + return ENOTCONN; + } + *source = mainloop_add_fd(crm_ipc_name(ipc), priority, crm_ipc_get_fd(ipc), + userdata, NULL); + if (*source == NULL) { + int rc = errno; - client = mainloop_add_fd(name, priority, fd, userdata, NULL); + crm_ipc_close(ipc); + return rc; } + (*source)->ipc = ipc; + (*source)->destroy_fn = callbacks->destroy; + (*source)->dispatch_fn_ipc = callbacks->dispatch; + return pcmk_rc_ok; +} - if (client == NULL) { - crm_perror(LOG_TRACE, "Connection to %s failed", name); - if (conn) { - crm_ipc_close(conn); - crm_ipc_destroy(conn); +mainloop_io_t * +mainloop_add_ipc_client(const char *name, int priority, size_t max_size, + void *userdata, struct ipc_client_callbacks *callbacks) +{ + crm_ipc_t *ipc = crm_ipc_new(name, max_size); + mainloop_io_t *source = NULL; + int rc = pcmk__add_mainloop_ipc(ipc, priority, userdata, callbacks, + &source); + + if (rc != pcmk_rc_ok) { + if (crm_log_level == LOG_STDOUT) { + fprintf(stderr, "Connection to %s failed: %s", + name, pcmk_rc_str(rc)); } + crm_ipc_destroy(ipc); return NULL; } - - client->ipc = conn; - client->destroy_fn = callbacks->destroy; - client->dispatch_fn_ipc = callbacks->dispatch; - return client; + return source; } void -- 1.8.3.1 From b9539da27998ff5e6c8b681f39603550a923ca33 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Mon, 6 Apr 2020 17:40:55 -0500 Subject: [PATCH 2/6] Refactor: libcrmcommon: add C API for controller IPC Implement a C API for controller IPC using the new IPC API model. --- include/crm/common/Makefile.am | 2 +- include/crm/common/ipc.h | 4 +- include/crm/common/ipc_controld.h | 99 +++++++ lib/common/Makefile.am | 1 + lib/common/crmcommon_private.h | 6 + lib/common/ipc_client.c | 46 +-- lib/common/ipc_controld.c | 609 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 723 insertions(+), 44 deletions(-) create mode 100644 include/crm/common/ipc_controld.h create mode 100644 lib/common/ipc_controld.c diff --git a/include/crm/common/Makefile.am b/include/crm/common/Makefile.am index 776e4a7..f29d105 100644 --- a/include/crm/common/Makefile.am +++ b/include/crm/common/Makefile.am @@ -12,7 +12,7 @@ MAINTAINERCLEANFILES = Makefile.in headerdir=$(pkgincludedir)/crm/common header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h results.h \ - nvpair.h acl.h + nvpair.h acl.h ipc_controld.h noinst_HEADERS = internal.h alerts_internal.h \ iso8601_internal.h remote_internal.h xml_internal.h \ ipc_internal.h output.h cmdline_internal.h curses_internal.h \ diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h index 8dee1b1..c67aaea 100644 --- a/include/crm/common/ipc.h +++ b/include/crm/common/ipc.h @@ -217,7 +217,9 @@ unsigned int crm_ipc_default_buffer_size(void); int crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid, pid_t *gotpid, uid_t *gotuid, gid_t *gotgid); -/* Utils */ +/* This is controller-specific but is declared in this header for C API + * backward compatibility. + */ xmlNode *create_hello_message(const char *uuid, const char *client_name, const char *major_version, const char *minor_version); diff --git a/include/crm/common/ipc_controld.h b/include/crm/common/ipc_controld.h new file mode 100644 index 0000000..0ebabfc --- /dev/null +++ b/include/crm/common/ipc_controld.h @@ -0,0 +1,99 @@ +/* + * Copyright 2020 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + +#ifndef PCMK__IPC_CONTROLD__H +# define PCMK__IPC_CONTROLD__H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \file + * \brief IPC commands for Pacemaker controller + * + * \ingroup core + */ + +#include // bool +#include // xmlNode +#include // pcmk_ipc_api_t + +//! Possible types of controller replies +enum pcmk_controld_api_reply { + pcmk_controld_reply_unknown, + pcmk_controld_reply_reprobe, + pcmk_controld_reply_info, + pcmk_controld_reply_resource, + pcmk_controld_reply_ping, +}; + +/*! + * Controller reply passed to event callback + * + * \note Shutdown and election calls have no reply. Reprobe calls are + * acknowledged but contain no data (reply_type will be the only item + * set). Node info and ping calls have their own reply data. Fail and + * refresh calls use the resource reply type and reply data. + * \note The pointers in the reply are only guaranteed to be meaningful for the + * execution of the callback; if the values are needed for later, the + * callback should copy them. + */ +typedef struct { + enum pcmk_controld_api_reply reply_type; + const char *feature_set; //!< CRM feature set advertised by controller + const char *host_from; //!< Name of node that sent reply + + union { + // pcmk_controld_reply_info + struct { + bool have_quorum; + bool is_remote; + int id; + const char *uuid; + const char *uname; + const char *state; + } node_info; + + // pcmk_controld_reply_resource + struct { + xmlNode *node_state; //cmds = pcmk__controld_api_methods(); break; case pcmk_ipc_execd: @@ -247,7 +249,7 @@ pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log) return for_log? "CIB manager" : NULL /* PCMK__SERVER_BASED_RW */; case pcmk_ipc_controld: - return for_log? "controller" : NULL /* CRM_SYSTEM_CRMD */; + return for_log? "controller" : CRM_SYSTEM_CRMD; case pcmk_ipc_execd: return for_log? "executor" : NULL /* CRM_SYSTEM_LRMD */; @@ -1412,43 +1414,3 @@ bail: } return rc; } - -xmlNode * -create_hello_message(const char *uuid, - const char *client_name, const char *major_version, const char *minor_version) -{ - xmlNode *hello_node = NULL; - xmlNode *hello = NULL; - - if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name) - || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) { - crm_err("Could not create IPC hello message from %s (UUID %s): " - "missing information", - client_name? client_name : "unknown client", - uuid? uuid : "unknown"); - return NULL; - } - - hello_node = create_xml_node(NULL, XML_TAG_OPTIONS); - if (hello_node == NULL) { - crm_err("Could not create IPC hello message from %s (UUID %s): " - "Message data creation failed", client_name, uuid); - return NULL; - } - - crm_xml_add(hello_node, "major_version", major_version); - crm_xml_add(hello_node, "minor_version", minor_version); - crm_xml_add(hello_node, "client_name", client_name); - crm_xml_add(hello_node, "client_uuid", uuid); - - hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid); - if (hello == NULL) { - crm_err("Could not create IPC hello message from %s (UUID %s): " - "Request creation failed", client_name, uuid); - return NULL; - } - free_xml(hello_node); - - crm_trace("Created hello message from %s (UUID %s)", client_name, uuid); - return hello; -} diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c new file mode 100644 index 0000000..22bb733 --- /dev/null +++ b/lib/common/ipc_controld.c @@ -0,0 +1,609 @@ +/* + * Copyright 2020 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include "crmcommon_private.h" + +struct controld_api_private_s { + char *client_uuid; + unsigned int replies_expected; +}; + +// \return Standard Pacemaker return code +static int +new_data(pcmk_ipc_api_t *api) +{ + struct controld_api_private_s *private = NULL; + + api->api_data = calloc(1, sizeof(struct controld_api_private_s)); + + if (api->api_data == NULL) { + return errno; + } + + private = api->api_data; + + /* This is set to the PID because that's how it was always done, but PIDs + * are not unique because clients can be remote. The value appears to be + * unused other than as part of F_CRM_SYS_FROM in IPC requests, which is + * only compared against the internal system names (CRM_SYSTEM_TENGINE, + * etc.), so it shouldn't be a problem. + */ + private->client_uuid = pcmk__getpid_s(); + + /* @TODO Implement a call ID model similar to the CIB, executor, and fencer + * IPC APIs, so that requests and replies can be matched, and + * duplicate replies can be discarded. + */ + return pcmk_rc_ok; +} + +static void +free_data(void *data) +{ + free(((struct controld_api_private_s *) data)->client_uuid); + free(data); +} + +// \return Standard Pacemaker return code +static int +post_connect(pcmk_ipc_api_t *api) +{ + /* The controller currently requires clients to register via a hello + * request, but does not reply back. + */ + struct controld_api_private_s *private = api->api_data; + const char *client_name = crm_system_name? crm_system_name : "client"; + xmlNode *hello; + int rc; + + hello = create_hello_message(private->client_uuid, client_name, + PCMK__CONTROLD_API_MAJOR, + PCMK__CONTROLD_API_MINOR); + rc = pcmk__send_ipc_request(api, hello); + free_xml(hello); + if (rc != pcmk_rc_ok) { + crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s", + pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc); + } else { + crm_debug("Sent IPC hello to %s", pcmk_ipc_name(api, true)); + } + return rc; +} + +#define xml_true(xml, field) crm_is_true(crm_element_value(xml, field)) + +static void +set_node_info_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data) +{ + data->reply_type = pcmk_controld_reply_info; + if (msg_data == NULL) { + return; + } + data->data.node_info.have_quorum = xml_true(msg_data, XML_ATTR_HAVE_QUORUM); + data->data.node_info.is_remote = xml_true(msg_data, XML_NODE_IS_REMOTE); + crm_element_value_int(msg_data, XML_ATTR_ID, &(data->data.node_info.id)); + data->data.node_info.uuid = crm_element_value(msg_data, XML_ATTR_UUID); + data->data.node_info.uname = crm_element_value(msg_data, XML_ATTR_UNAME); + data->data.node_info.state = crm_element_value(msg_data, XML_NODE_IS_PEER); +} + +static void +set_ping_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data) +{ + data->reply_type = pcmk_controld_reply_ping; + if (msg_data == NULL) { + return; + } + data->data.ping.sys_from = crm_element_value(msg_data, + XML_PING_ATTR_SYSFROM); + data->data.ping.fsa_state = crm_element_value(msg_data, + XML_PING_ATTR_CRMDSTATE); + data->data.ping.result = crm_element_value(msg_data, XML_PING_ATTR_STATUS); +} + +static bool +reply_expected(pcmk_ipc_api_t *api, xmlNode *request) +{ + const char *command = crm_element_value(request, F_CRM_TASK); + + if (command == NULL) { + return false; + } + + // We only need to handle commands that functions in this file can send + return !strcmp(command, CRM_OP_REPROBE) + || !strcmp(command, CRM_OP_NODE_INFO) + || !strcmp(command, CRM_OP_PING) + || !strcmp(command, CRM_OP_LRM_FAIL) + || !strcmp(command, CRM_OP_LRM_DELETE); +} + +static void +dispatch(pcmk_ipc_api_t *api, xmlNode *reply) +{ + struct controld_api_private_s *private = api->api_data; + crm_exit_t status = CRM_EX_OK; + xmlNode *msg_data = NULL; + const char *value = NULL; + pcmk_controld_api_reply_t reply_data = { + pcmk_controld_reply_unknown, NULL, NULL, + }; + + if (private->replies_expected > 0) { + private->replies_expected--; + } + + // Do some basic validation of the reply + + /* @TODO We should be able to verify that value is always a response, but + * currently the controller doesn't always properly set the type. Even + * if we fix the controller, we'll still need to handle replies from + * old versions (feature set could be used to differentiate). + */ + value = crm_element_value(reply, F_CRM_MSG_TYPE); + if ((value == NULL) || (strcmp(value, XML_ATTR_REQUEST) + && strcmp(value, XML_ATTR_RESPONSE))) { + crm_debug("Unrecognizable controller message: invalid message type '%s'", + crm_str(value)); + status = CRM_EX_PROTOCOL; + reply = NULL; + } + + if (crm_element_value(reply, XML_ATTR_REFERENCE) == NULL) { + crm_debug("Unrecognizable controller message: no reference"); + status = CRM_EX_PROTOCOL; + reply = NULL; + } + + value = crm_element_value(reply, F_CRM_TASK); + if (value == NULL) { + crm_debug("Unrecognizable controller message: no command name"); + status = CRM_EX_PROTOCOL; + reply = NULL; + } + + // Parse useful info from reply + + if (reply != NULL) { + reply_data.feature_set = crm_element_value(reply, XML_ATTR_VERSION); + reply_data.host_from = crm_element_value(reply, F_CRM_HOST_FROM); + msg_data = get_message_xml(reply, F_CRM_DATA); + + if (!strcmp(value, CRM_OP_REPROBE)) { + reply_data.reply_type = pcmk_controld_reply_reprobe; + + } else if (!strcmp(value, CRM_OP_NODE_INFO)) { + set_node_info_data(&reply_data, msg_data); + + } else if (!strcmp(value, CRM_OP_INVOKE_LRM)) { + reply_data.reply_type = pcmk_controld_reply_resource; + reply_data.data.resource.node_state = msg_data; + + } else if (!strcmp(value, CRM_OP_PING)) { + set_ping_data(&reply_data, msg_data); + + } else { + crm_debug("Unrecognizable controller message: unknown command '%s'", + value); + status = CRM_EX_PROTOCOL; + reply = NULL; + } + } + + pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data); +} + +pcmk__ipc_methods_t * +pcmk__controld_api_methods() +{ + pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t)); + + if (cmds != NULL) { + cmds->new_data = new_data; + cmds->free_data = free_data; + cmds->post_connect = post_connect; + cmds->reply_expected = reply_expected; + cmds->dispatch = dispatch; + } + return cmds; +} + +/*! + * \internal + * \brief Create XML for a controller IPC request + * + * \param[in] api Controller connection + * \param[in] op Controller IPC command name + * \param[in] node Node name to set as destination host + * \param[in] msg_data XML to attach to request as message data + * + * \return Newly allocated XML for request + */ +static xmlNode * +create_controller_request(pcmk_ipc_api_t *api, const char *op, + const char *node, xmlNode *msg_data) +{ + struct controld_api_private_s *private = api->api_data; + const char *sys_to = NULL; + + if ((node == NULL) && !strcmp(op, CRM_OP_PING)) { + sys_to = CRM_SYSTEM_DC; + } else { + sys_to = CRM_SYSTEM_CRMD; + } + return create_request(op, msg_data, node, sys_to, + (crm_system_name? crm_system_name : "client"), + private->client_uuid); +} + +// \return Standard Pacemaker return code +static int +send_controller_request(pcmk_ipc_api_t *api, xmlNode *request, + bool reply_is_expected) +{ + int rc; + + if (crm_element_value(request, XML_ATTR_REFERENCE) == NULL) { + return EINVAL; + } + rc = pcmk__send_ipc_request(api, request); + if ((rc == pcmk_rc_ok) && reply_is_expected) { + struct controld_api_private_s *private = api->api_data; + + private->replies_expected++; + } + return rc; +} + +static xmlNode * +create_reprobe_message_data(const char *target_node, const char *router_node) +{ + xmlNode *msg_data; + + msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE); + crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node); + if ((router_node != NULL) && safe_str_neq(router_node, target_node)) { + crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node); + } + return msg_data; +} + +/*! + * \brief Send a reprobe controller operation + * + * \param[in] api Controller connection + * \param[in] target_node Name of node to reprobe + * \param[in] router_node Router node for host + * + * \return Standard Pacemaker return code + * \note Event callback will get a reply of type pcmk_controld_reply_reprobe. + */ +int +pcmk_controld_api_reprobe(pcmk_ipc_api_t *api, const char *target_node, + const char *router_node) +{ + xmlNode *request; + xmlNode *msg_data; + int rc = pcmk_rc_ok; + + if (api == NULL) { + return EINVAL; + } + if (router_node == NULL) { + router_node = target_node; + } + crm_debug("Sending %s IPC request to reprobe %s via %s", + pcmk_ipc_name(api, true), crm_str(target_node), + crm_str(router_node)); + msg_data = create_reprobe_message_data(target_node, router_node); + request = create_controller_request(api, CRM_OP_REPROBE, router_node, + msg_data); + rc = send_controller_request(api, request, true); + free_xml(msg_data); + free_xml(request); + return rc; +} + +/*! + * \brief Send a "node info" controller operation + * + * \param[in] api Controller connection + * \param[in] nodeid ID of node to get info for (or 0 for local node) + * + * \return Standard Pacemaker return code + * \note Event callback will get a reply of type pcmk_controld_reply_info. + */ +int +pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid) +{ + xmlNode *request; + int rc = pcmk_rc_ok; + + request = create_controller_request(api, CRM_OP_NODE_INFO, NULL, NULL); + if (request == NULL) { + return EINVAL; + } + if (nodeid > 0) { + crm_xml_set_id(request, "%lu", (unsigned long) nodeid); + } + + rc = send_controller_request(api, request, true); + free_xml(request); + return rc; +} + +/*! + * \brief Ask the controller for status + * + * \param[in] api Controller connection + * \param[in] node_name Name of node whose status is desired (or NULL for DC) + * + * \return Standard Pacemaker return code + * \note Event callback will get a reply of type pcmk_controld_reply_ping. + */ +int +pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name) +{ + xmlNode *request; + int rc = pcmk_rc_ok; + + request = create_controller_request(api, CRM_OP_PING, node_name, NULL); + if (request == NULL) { + return EINVAL; + } + rc = send_controller_request(api, request, true); + free_xml(request); + return rc; +} + +/*! + * \internal + * \brief Ask the controller to shut down + * + * \param[in] api Controller connection + * \param[in] node_name Name of node whose controller should shut down + * + * \return Standard Pacemaker return code + * + * \note This capability currently does not work, so the function is considered + * internal. It will likely be removed. + * \note Event callback will not get a reply. + */ +int +pcmk_controld_api_shutdown(pcmk_ipc_api_t *api, const char *node_name) +{ + xmlNode *request; + int rc = pcmk_rc_ok; + + request = create_controller_request(api, CRM_OP_SHUTDOWN, NULL, NULL); + if (request == NULL) { + return EINVAL; + } + rc = send_controller_request(api, request, false); + free_xml(request); + return rc; +} + +/*! + * \internal + * \brief Ask the controller to start a DC election + * + * \param[in] api Controller connection + * + * \return Standard Pacemaker return code + * + * \note This capability currently does not work, so the function is considered + * internal. It will likely be removed. + * \note Event callback will not get a reply. + */ +int +pcmk_controld_api_start_election(pcmk_ipc_api_t *api) +{ + xmlNode *request; + int rc = pcmk_rc_ok; + + request = create_controller_request(api, CRM_OP_VOTE, NULL, NULL); + if (request == NULL) { + return EINVAL; + } + rc = send_controller_request(api, request, false); + free_xml(request); + return rc; +} + +// \return Standard Pacemaker return code +static int +controller_resource_op(pcmk_ipc_api_t *api, const char *op, + const char *target_node, const char *router_node, + bool cib_only, const char *rsc_id, + const char *rsc_long_id, const char *standard, + const char *provider, const char *type) +{ + int rc = pcmk_rc_ok; + char *key; + xmlNode *request, *msg_data, *xml_rsc, *params; + + if (api == NULL) { + return EINVAL; + } + if (router_node == NULL) { + router_node = target_node; + } + + msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP); + + /* The controller logs the transition key from resource op requests, so we + * need to have *something* for it. + * @TODO don't use "crm-resource" + */ + key = pcmk__transition_key(0, getpid(), 0, + "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx"); + crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key); + free(key); + + crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node); + if (safe_str_neq(router_node, target_node)) { + crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node); + } + + if (cib_only) { + // Indicate that only the CIB needs to be cleaned + crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB); + } + + xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE); + crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id); + crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc_long_id); + crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, standard); + crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, provider); + crm_xml_add(xml_rsc, XML_ATTR_TYPE, type); + + params = create_xml_node(msg_data, XML_TAG_ATTRS); + crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET); + + // The controller parses the timeout from the request + key = crm_meta_name(XML_ATTR_TIMEOUT); + crm_xml_add(params, key, "60000"); /* 1 minute */ //@TODO pass as arg + free(key); + + request = create_controller_request(api, op, router_node, msg_data); + rc = send_controller_request(api, request, true); + free_xml(msg_data); + free_xml(request); + return rc; +} + +/*! + * \brief Ask the controller to fail a resource + * + * \param[in] api Controller connection + * \param[in] target_node Name of node resource is on + * \param[in] router_node Router node for target + * \param[in] rsc_id ID of resource to fail + * \param[in] rsc_long_id Long ID of resource (if any) + * \param[in] standard Standard of resource + * \param[in] provider Provider of resource (if any) + * \param[in] type Type of resource to fail + * + * \return Standard Pacemaker return code + * \note Event callback will get a reply of type pcmk_controld_reply_resource. + */ +int +pcmk_controld_api_fail(pcmk_ipc_api_t *api, + const char *target_node, const char *router_node, + const char *rsc_id, const char *rsc_long_id, + const char *standard, const char *provider, + const char *type) +{ + crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s", + pcmk_ipc_name(api, true), crm_str(rsc_id), crm_str(rsc_long_id), + crm_str(target_node), crm_str(router_node)); + return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node, + router_node, false, rsc_id, rsc_long_id, + standard, provider, type); +} + +/*! + * \brief Ask the controller to refresh a resource + * + * \param[in] api Controller connection + * \param[in] target_node Name of node resource is on + * \param[in] router_node Router node for target + * \param[in] rsc_id ID of resource to refresh + * \param[in] rsc_long_id Long ID of resource (if any) + * \param[in] standard Standard of resource + * \param[in] provider Provider of resource (if any) + * \param[in] type Type of resource + * \param[in] cib_only If true, clean resource from CIB only + * + * \return Standard Pacemaker return code + * \note Event callback will get a reply of type pcmk_controld_reply_resource. + */ +int +pcmk_controld_api_refresh(pcmk_ipc_api_t *api, const char *target_node, + const char *router_node, + const char *rsc_id, const char *rsc_long_id, + const char *standard, const char *provider, + const char *type, bool cib_only) +{ + crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s", + pcmk_ipc_name(api, true), crm_str(rsc_id), crm_str(rsc_long_id), + crm_str(target_node), crm_str(router_node)); + return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node, + router_node, cib_only, rsc_id, rsc_long_id, + standard, provider, type); +} + +/*! + * \brief Get the number of IPC replies currently expected from the controller + * + * \param[in] api Controller IPC API connection + * + * \return Number of replies expected + */ +unsigned int +pcmk_controld_api_replies_expected(pcmk_ipc_api_t *api) +{ + struct controld_api_private_s *private = api->api_data; + + return private->replies_expected; +} + +xmlNode * +create_hello_message(const char *uuid, const char *client_name, + const char *major_version, const char *minor_version) +{ + xmlNode *hello_node = NULL; + xmlNode *hello = NULL; + + if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name) + || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) { + crm_err("Could not create IPC hello message from %s (UUID %s): " + "missing information", + client_name? client_name : "unknown client", + uuid? uuid : "unknown"); + return NULL; + } + + hello_node = create_xml_node(NULL, XML_TAG_OPTIONS); + if (hello_node == NULL) { + crm_err("Could not create IPC hello message from %s (UUID %s): " + "Message data creation failed", client_name, uuid); + return NULL; + } + + crm_xml_add(hello_node, "major_version", major_version); + crm_xml_add(hello_node, "minor_version", minor_version); + crm_xml_add(hello_node, "client_name", client_name); + crm_xml_add(hello_node, "client_uuid", uuid); + + hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid); + if (hello == NULL) { + crm_err("Could not create IPC hello message from %s (UUID %s): " + "Request creation failed", client_name, uuid); + return NULL; + } + free_xml(hello_node); + + crm_trace("Created hello message from %s (UUID %s)", client_name, uuid); + return hello; +} -- 1.8.3.1 From 1d1b34664b64f6805aeaf4d8e29e77aa8f59b4fc Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 7 Apr 2020 15:43:01 -0500 Subject: [PATCH 3/6] Refactor: tools: convert crm_resource to use new controller IPC model --- tools/Makefile.am | 3 +- tools/crm_resource.c | 138 +++++++------ tools/crm_resource.h | 9 +- tools/crm_resource_controller.c | 425 ---------------------------------------- tools/crm_resource_controller.h | 198 ------------------- tools/crm_resource_runtime.c | 30 +-- 6 files changed, 96 insertions(+), 707 deletions(-) delete mode 100644 tools/crm_resource_controller.c delete mode 100644 tools/crm_resource_controller.h diff --git a/tools/Makefile.am b/tools/Makefile.am index c822a8c..4609b0f 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -12,7 +12,7 @@ if BUILD_SYSTEMD systemdsystemunit_DATA = crm_mon.service endif -noinst_HEADERS = crm_mon.h crm_resource.h crm_resource_controller.h +noinst_HEADERS = crm_mon.h crm_resource.h pcmkdir = $(datadir)/$(PACKAGE) pcmk_DATA = report.common report.collector @@ -115,7 +115,6 @@ crm_attribute_LDADD = $(top_builddir)/lib/cluster/libcrmcluster.la \ crm_resource_SOURCES = crm_resource.c \ crm_resource_ban.c \ - crm_resource_controller.c \ crm_resource_print.c \ crm_resource_runtime.c crm_resource_LDADD = $(top_builddir)/lib/pengine/libpe_rules.la \ diff --git a/tools/crm_resource.c b/tools/crm_resource.c index 6853ad5..c8c1cfa 100644 --- a/tools/crm_resource.c +++ b/tools/crm_resource.c @@ -11,62 +11,70 @@ #include #include - -#include -#include - #include #include #include - #include #include #include #include #include +#include +#include +#include + bool BE_QUIET = FALSE; bool scope_master = FALSE; int cib_options = cib_sync_call; - -static GMainLoop *mainloop = NULL; +static crm_exit_t exit_code = CRM_EX_OK; // Things that should be cleaned up on exit +static GMainLoop *mainloop = NULL; static cib_t *cib_conn = NULL; -static pcmk_controld_api_t *controld_api = NULL; +static pcmk_ipc_api_t *controld_api = NULL; static pe_working_set_t *data_set = NULL; #define MESSAGE_TIMEOUT_S 60 // Clean up and exit static crm_exit_t -bye(crm_exit_t exit_code) +bye(crm_exit_t ec) { - static bool crm_resource_shutdown_flag = false; - - if (crm_resource_shutdown_flag) { - // Allow usage like "return bye(...);" - return exit_code; - } - crm_resource_shutdown_flag = true; - if (cib_conn != NULL) { cib_t *save_cib_conn = cib_conn; - cib_conn = NULL; + cib_conn = NULL; // Ensure we can't free this twice save_cib_conn->cmds->signoff(save_cib_conn); cib_delete(save_cib_conn); } if (controld_api != NULL) { - pcmk_controld_api_t *save_controld_api = controld_api; + pcmk_ipc_api_t *save_controld_api = controld_api; - controld_api = NULL; - pcmk_free_controld_api(save_controld_api); + controld_api = NULL; // Ensure we can't free this twice + pcmk_free_ipc_api(save_controld_api); + } + if (mainloop != NULL) { + g_main_loop_unref(mainloop); + mainloop = NULL; } pe_free_working_set(data_set); data_set = NULL; - crm_exit(exit_code); - return exit_code; + crm_exit(ec); + return ec; +} + +static void +quit_main_loop(crm_exit_t ec) +{ + exit_code = ec; + if (mainloop != NULL) { + GMainLoop *mloop = mainloop; + + mainloop = NULL; // Don't re-enter this block + pcmk_quit_main_loop(mloop, 10); + g_main_loop_unref(mloop); + } } static gboolean @@ -76,39 +84,54 @@ resource_ipc_timeout(gpointer data) fprintf(stderr, "\nAborting because no messages received in %d seconds\n", MESSAGE_TIMEOUT_S); crm_err("No messages received in %d seconds", MESSAGE_TIMEOUT_S); - bye(CRM_EX_TIMEOUT); + quit_main_loop(CRM_EX_TIMEOUT); return FALSE; } static void -handle_controller_reply(pcmk_controld_api_t *capi, void *api_data, - void *user_data) +controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type, + crm_exit_t status, void *event_data, void *user_data) { - fprintf(stderr, "."); - if ((capi->replies_expected(capi) == 0) - && mainloop && g_main_loop_is_running(mainloop)) { - fprintf(stderr, " OK\n"); - crm_debug("Got all the replies we expected"); - bye(CRM_EX_OK); - } -} + switch (event_type) { + case pcmk_ipc_event_disconnect: + if (exit_code == CRM_EX_DISCONNECT) { // Unexpected + crm_info("Connection to controller was terminated"); + } + quit_main_loop(exit_code); + break; -static void -handle_controller_drop(pcmk_controld_api_t *capi, void *api_data, - void *user_data) -{ - crm_info("Connection to controller was terminated"); - bye(CRM_EX_DISCONNECT); + case pcmk_ipc_event_reply: + if (status != CRM_EX_OK) { + fprintf(stderr, "\nError: bad reply from controller: %s\n", + crm_exit_str(status)); + pcmk_disconnect_ipc(api); + quit_main_loop(status); + } else { + fprintf(stderr, "."); + if ((pcmk_controld_api_replies_expected(api) == 0) + && mainloop && g_main_loop_is_running(mainloop)) { + fprintf(stderr, " OK\n"); + crm_debug("Got all the replies we expected"); + pcmk_disconnect_ipc(api); + quit_main_loop(CRM_EX_OK); + } + } + break; + + default: + break; + } } static void -start_mainloop(pcmk_controld_api_t *capi) +start_mainloop(pcmk_ipc_api_t *capi) { - if (capi->replies_expected(capi) > 0) { - unsigned int count = capi->replies_expected(capi); + unsigned int count = pcmk_controld_api_replies_expected(capi); + if (count > 0) { fprintf(stderr, "Waiting for %d %s from the controller", count, pcmk__plural_alt(count, "reply", "replies")); + exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects mainloop = g_main_loop_new(NULL, FALSE); g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL); g_main_loop_run(mainloop); @@ -664,7 +687,6 @@ main(int argc, char **argv) int argerr = 0; int flag; int find_flags = 0; // Flags to use when searching for resource - crm_exit_t exit_code = CRM_EX_OK; crm_log_cli_init("crm_resource"); pcmk__set_cli_options(NULL, "| [options]", long_options, @@ -1151,21 +1173,15 @@ main(int argc, char **argv) // Establish a connection to the controller if needed if (require_crmd) { - char *client_uuid; - pcmk_controld_api_cb_t dispatch_cb = { - handle_controller_reply, NULL - }; - pcmk_controld_api_cb_t destroy_cb = { - handle_controller_drop, NULL - }; - - - client_uuid = pcmk__getpid_s(); - controld_api = pcmk_new_controld_api(crm_system_name, client_uuid); - free(client_uuid); - - rc = controld_api->connect(controld_api, true, &dispatch_cb, - &destroy_cb); + rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld); + if (rc != pcmk_rc_ok) { + CMD_ERR("Error connecting to the controller: %s", pcmk_rc_str(rc)); + rc = pcmk_rc2legacy(rc); + goto bail; + } + pcmk_register_ipc_callback(controld_api, controller_event_callback, + NULL); + rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main); if (rc != pcmk_rc_ok) { CMD_ERR("Error connecting to the controller: %s", pcmk_rc_str(rc)); rc = pcmk_rc2legacy(rc); @@ -1525,8 +1541,8 @@ main(int argc, char **argv) NULL, NULL, NULL, NULL, attr_options)); - if (controld_api->reprobe(controld_api, host_uname, - router_node) == pcmk_rc_ok) { + if (pcmk_controld_api_reprobe(controld_api, host_uname, + router_node) == pcmk_rc_ok) { start_mainloop(controld_api); } diff --git a/tools/crm_resource.h b/tools/crm_resource.h index 0bf7bee..cb7f506 100644 --- a/tools/crm_resource.h +++ b/tools/crm_resource.h @@ -21,7 +21,6 @@ #include #include #include -#include "crm_resource_controller.h" extern bool print_pending; @@ -36,8 +35,6 @@ extern char *move_lifetime; extern const char *attr_set_type; -extern pcmk_controld_api_cb_t controld_api_cb; - /* ban */ int cli_resource_prefer(const char *rsc_id, const char *host, cib_t * cib_conn); int cli_resource_ban(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn); @@ -63,16 +60,16 @@ int cli_resource_print_operations(const char *rsc_id, const char *host_uname, bo /* runtime */ void cli_resource_check(cib_t * cib, pe_resource_t *rsc); -int cli_resource_fail(pcmk_controld_api_t *controld_api, +int cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname, const char *rsc_id, pe_working_set_t *data_set); int cli_resource_search(pe_resource_t *rsc, const char *requested_name, pe_working_set_t *data_set); -int cli_resource_delete(pcmk_controld_api_t *controld_api, +int cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname, pe_resource_t *rsc, const char *operation, const char *interval_spec, bool just_failures, pe_working_set_t *data_set); -int cli_cleanup_all(pcmk_controld_api_t *controld_api, const char *node_name, +int cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name, const char *operation, const char *interval_spec, pe_working_set_t *data_set); int cli_resource_restart(pe_resource_t *rsc, const char *host, int timeout_ms, diff --git a/tools/crm_resource_controller.c b/tools/crm_resource_controller.c deleted file mode 100644 index 994a7be..0000000 --- a/tools/crm_resource_controller.c +++ /dev/null @@ -1,425 +0,0 @@ -/* - * Copyright 2020 the Pacemaker project contributors - * - * The version control history for this file may have further details. - * - * This source code is licensed under the GNU General Public License version 2 - * or later (GPLv2+) WITHOUT ANY WARRANTY. - */ - -#include -#include -#include -#include "crm_resource.h" - -// API object's private members -struct controller_private { - char *client_name; // Client name to use with IPC - char *client_uuid; // Client UUID to use with IPC - mainloop_io_t *source; // If main loop used, I/O source for IPC - crm_ipc_t *ipc; // IPC connection to controller - int replies_expected; // How many controller replies are expected - pcmk_controld_api_cb_t dispatch_cb; // Caller's registered dispatch callback - pcmk_controld_api_cb_t destroy_cb; // Caller's registered destroy callback -}; - -static void -call_client_callback(pcmk_controld_api_t *api, pcmk_controld_api_cb_t *cb, - void *api_data) -{ - if ((cb != NULL) && (cb->callback != NULL)) { - cb->callback(api, api_data, cb->user_data); - } -} - -/* - * IPC callbacks when used with main loop - */ - -static void -controller_ipc_destroy(gpointer user_data) -{ - pcmk_controld_api_t *api = user_data; - struct controller_private *private = api->private; - - private->ipc = NULL; - private->source = NULL; - call_client_callback(api, &(private->destroy_cb), NULL); -} - -// \return < 0 if connection is no longer required, >= 0 if it is -static int -controller_ipc_dispatch(const char *buffer, ssize_t length, gpointer user_data) -{ - xmlNode *msg = NULL; - pcmk_controld_api_t *api = user_data; - - CRM_CHECK(buffer && api && api->private, return 0); - - msg = string2xml(buffer); - if (msg == NULL) { - crm_warn("Received malformed controller IPC message"); - } else { - struct controller_private *private = api->private; - - crm_log_xml_trace(msg, "controller-reply"); - private->replies_expected--; - call_client_callback(api, &(private->dispatch_cb), - get_message_xml(msg, F_CRM_DATA)); - free_xml(msg); - } - return 0; -} - -/* - * IPC utilities - */ - -// \return Standard Pacemaker return code -static int -send_hello(crm_ipc_t *ipc, const char *client_name, const char *client_uuid) -{ - xmlNode *hello = create_hello_message(client_uuid, client_name, "0", "1"); - int rc = crm_ipc_send(ipc, hello, 0, 0, NULL); - - free_xml(hello); - if (rc < 0) { - rc = pcmk_legacy2rc(rc); - crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s", - CRM_SYSTEM_CRMD /* ipc->name */, - pcmk_rc_str(rc), rc); - return rc; - } - crm_debug("Sent IPC hello to %s", CRM_SYSTEM_CRMD /* ipc->name */); - return pcmk_rc_ok; -} - -// \return Standard Pacemaker return code -static int -send_controller_request(pcmk_controld_api_t *api, const char *op, - xmlNode *msg_data, const char *node) -{ - int rc; - struct controller_private *private = api->private; - xmlNode *cmd = create_request(op, msg_data, node, CRM_SYSTEM_CRMD, - private->client_name, private->client_uuid); - const char *reference = crm_element_value(cmd, XML_ATTR_REFERENCE); - - if ((cmd == NULL) || (reference == NULL)) { - return EINVAL; - } - - //@TODO pass as args? 0=crm_ipc_flags, 0=timeout_ms (default 5s), NULL=reply - crm_log_xml_trace(cmd, "controller-request"); - rc = crm_ipc_send(private->ipc, cmd, 0, 0, NULL); - free_xml(cmd); - if (rc < 0) { - return pcmk_legacy2rc(rc); - } - private->replies_expected++; - return pcmk_rc_ok; -} - -/* - * pcmk_controld_api_t methods - */ - -static int -controller_connect_mainloop(pcmk_controld_api_t *api) -{ - struct controller_private *private = api->private; - struct ipc_client_callbacks callbacks = { - .dispatch = controller_ipc_dispatch, - .destroy = controller_ipc_destroy, - }; - - private->source = mainloop_add_ipc_client(CRM_SYSTEM_CRMD, - G_PRIORITY_DEFAULT, 0, api, - &callbacks); - if (private->source == NULL) { - return ENOTCONN; - } - - private->ipc = mainloop_get_ipc_client(private->source); - if (private->ipc == NULL) { - (void) api->disconnect(api); - return ENOTCONN; - } - - crm_debug("Connected to %s IPC (attaching to main loop)", CRM_SYSTEM_CRMD); - return pcmk_rc_ok; -} - -static int -controller_connect_no_mainloop(pcmk_controld_api_t *api) -{ - struct controller_private *private = api->private; - - private->ipc = crm_ipc_new(CRM_SYSTEM_CRMD, 0); - if (private->ipc == NULL) { - return ENOTCONN; - } - if (!crm_ipc_connect(private->ipc)) { - crm_ipc_close(private->ipc); - crm_ipc_destroy(private->ipc); - private->ipc = NULL; - return errno; - } - /* @TODO caller needs crm_ipc_get_fd(private->ipc); either add method for - * that, or replace use_mainloop with int *fd - */ - crm_debug("Connected to %s IPC", CRM_SYSTEM_CRMD); - return pcmk_rc_ok; -} - -static void -set_callback(pcmk_controld_api_cb_t *dest, pcmk_controld_api_cb_t *source) -{ - if (source) { - dest->callback = source->callback; - dest->user_data = source->user_data; - } -} - -static int -controller_api_connect(pcmk_controld_api_t *api, bool use_mainloop, - pcmk_controld_api_cb_t *dispatch_cb, - pcmk_controld_api_cb_t *destroy_cb) -{ - int rc = pcmk_rc_ok; - struct controller_private *private; - - if (api == NULL) { - return EINVAL; - } - private = api->private; - - set_callback(&(private->dispatch_cb), dispatch_cb); - set_callback(&(private->destroy_cb), destroy_cb); - - if (private->ipc != NULL) { - return pcmk_rc_ok; // already connected - } - - if (use_mainloop) { - rc = controller_connect_mainloop(api); - } else { - rc = controller_connect_no_mainloop(api); - } - if (rc != pcmk_rc_ok) { - return rc; - } - - rc = send_hello(private->ipc, private->client_name, private->client_uuid); - if (rc != pcmk_rc_ok) { - (void) api->disconnect(api); - } - return rc; -} - -static int -controller_api_disconnect(pcmk_controld_api_t *api) -{ - struct controller_private *private = api->private; - - if (private->source != NULL) { - // Attached to main loop - mainloop_del_ipc_client(private->source); - private->source = NULL; - private->ipc = NULL; - - } else if (private->ipc != NULL) { - // Not attached to main loop - crm_ipc_t *ipc = private->ipc; - - private->ipc = NULL; - crm_ipc_close(ipc); - crm_ipc_destroy(ipc); - } - crm_debug("Disconnected from %s IPC", CRM_SYSTEM_CRMD /* ipc->name */); - return pcmk_rc_ok; -} - -//@TODO dispatch function for non-mainloop a la stonith_dispatch() -//@TODO convenience retry-connect function a la stonith_api_connect_retry() - -static unsigned int -controller_api_replies_expected(pcmk_controld_api_t *api) -{ - if (api != NULL) { - struct controller_private *private = api->private; - - return private->replies_expected; - } - return 0; -} - -static xmlNode * -create_reprobe_message_data(const char *target_node, const char *router_node) -{ - xmlNode *msg_data; - - msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE); - crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node); - if ((router_node != NULL) && safe_str_neq(router_node, target_node)) { - crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node); - } - return msg_data; -} - -static int -controller_api_reprobe(pcmk_controld_api_t *api, const char *target_node, - const char *router_node) -{ - int rc = EINVAL; - - if (api != NULL) { - xmlNode *msg_data; - - crm_debug("Sending %s IPC request to reprobe %s via %s", - CRM_SYSTEM_CRMD, crm_str(target_node), crm_str(router_node)); - msg_data = create_reprobe_message_data(target_node, router_node); - rc = send_controller_request(api, CRM_OP_REPROBE, msg_data, - (router_node? router_node : target_node)); - free_xml(msg_data); - } - return rc; -} - -// \return Standard Pacemaker return code -static int -controller_resource_op(pcmk_controld_api_t *api, const char *op, - const char *target_node, const char *router_node, - bool cib_only, const char *rsc_id, - const char *rsc_long_id, const char *standard, - const char *provider, const char *type) -{ - int rc; - char *key; - xmlNode *msg_data, *xml_rsc, *params; - - if (api == NULL) { - return EINVAL; - } - if (router_node == NULL) { - router_node = target_node; - } - - msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP); - - /* The controller logs the transition key from resource op requests, so we - * need to have *something* for it. - */ - key = pcmk__transition_key(0, getpid(), 0, - "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx"); - crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key); - free(key); - - crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node); - if (safe_str_neq(router_node, target_node)) { - crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node); - } - - if (cib_only) { - // Indicate that only the CIB needs to be cleaned - crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB); - } - - xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE); - crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id); - crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc_long_id); - crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, standard); - crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, provider); - crm_xml_add(xml_rsc, XML_ATTR_TYPE, type); - - params = create_xml_node(msg_data, XML_TAG_ATTRS); - crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET); - - // The controller parses the timeout from the request - key = crm_meta_name(XML_ATTR_TIMEOUT); - crm_xml_add(params, key, "60000"); /* 1 minute */ //@TODO pass as arg - free(key); - - rc = send_controller_request(api, op, msg_data, router_node); - free_xml(msg_data); - return rc; -} - -static int -controller_api_fail_resource(pcmk_controld_api_t *api, - const char *target_node, const char *router_node, - const char *rsc_id, const char *rsc_long_id, - const char *standard, const char *provider, - const char *type) -{ - crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s", - CRM_SYSTEM_CRMD, crm_str(rsc_id), crm_str(rsc_long_id), - crm_str(target_node), crm_str(router_node)); - return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node, - router_node, false, rsc_id, rsc_long_id, - standard, provider, type); -} - -static int -controller_api_refresh_resource(pcmk_controld_api_t *api, - const char *target_node, - const char *router_node, - const char *rsc_id, const char *rsc_long_id, - const char *standard, const char *provider, - const char *type, bool cib_only) -{ - crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s", - CRM_SYSTEM_CRMD, crm_str(rsc_id), crm_str(rsc_long_id), - crm_str(target_node), crm_str(router_node)); - return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node, - router_node, cib_only, rsc_id, rsc_long_id, - standard, provider, type); -} - -pcmk_controld_api_t * -pcmk_new_controld_api(const char *client_name, const char *client_uuid) -{ - struct controller_private *private; - pcmk_controld_api_t *api = calloc(1, sizeof(pcmk_controld_api_t)); - - CRM_ASSERT(api != NULL); - - api->private = calloc(1, sizeof(struct controller_private)); - CRM_ASSERT(api->private != NULL); - private = api->private; - - if (client_name == NULL) { - client_name = crm_system_name? crm_system_name : "client"; - } - private->client_name = strdup(client_name); - CRM_ASSERT(private->client_name != NULL); - - if (client_uuid == NULL) { - private->client_uuid = crm_generate_uuid(); - } else { - private->client_uuid = strdup(client_uuid); - } - CRM_ASSERT(private->client_uuid != NULL); - - api->connect = controller_api_connect; - api->disconnect = controller_api_disconnect; - api->replies_expected = controller_api_replies_expected; - api->reprobe = controller_api_reprobe; - api->fail_resource = controller_api_fail_resource; - api->refresh_resource = controller_api_refresh_resource; - return api; -} - -void -pcmk_free_controld_api(pcmk_controld_api_t *api) -{ - if (api != NULL) { - struct controller_private *private = api->private; - - api->disconnect(api); - free(private->client_name); - free(private->client_uuid); - free(api->private); - free(api); - } -} diff --git a/tools/crm_resource_controller.h b/tools/crm_resource_controller.h deleted file mode 100644 index 50e20b4..0000000 --- a/tools/crm_resource_controller.h +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright 2020 the Pacemaker project contributors - * - * The version control history for this file may have further details. - * - * This source code is licensed under the GNU Lesser General Public License - * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. - */ -#ifndef PCMK__CONTROLD_API_H -#define PCMK__CONTROLD_API_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include // bool - -/* This is a demonstration of an abstracted controller IPC API. It is expected - * that this will be improved and moved to libcrmcommon. - * - * @TODO We could consider whether it's reasonable to have a single type for - * all daemons' IPC APIs (e.g. pcmk_ipc_api_t instead of pcmk_*_api_t). They - * could potentially have common connect/disconnect methods and then a void* to - * a group of API-specific methods. - * - * In that case, the callback type would also need to be generic, taking - * (pcmk_ipc_api_t *api, void *api_data, void *user_data), with individual APIs - * having functions for getting useful info from api_data. If all APIs followed - * the call_id model, we could use int call_id instead of api_data. - * - * A major annoyance is that the controller IPC protocol currently does not have - * any way to tie a particular reply to a particular request. The current - * clients (crmadmin, crm_node, and crm_resource) simply know what kind of reply - * to expect for the kind of request they sent. In crm_resource's case, all it - * does is count replies, ignoring their content altogether. - * - * That really forces us to have a single callback for all events rather than a - * per-request callback. That in turn implies that callers can only provide a - * single user data pointer. - * - * @TODO Define protocol version constants to use in hello message. - * @TODO Allow callers to specify timeouts. - * @TODO Define call IDs for controller ops, while somehow maintaining backward - * compatibility, since a client running on a Pacemaker Remote node could - * be older or newer than the controller on the connection's cluster - * node. - * @TODO The controller currently does not respond to hello messages. We should - * establish a common connection handshake protocol for all daemons that - * involves a hello message and acknowledgement. We should support sync - * or async connection (i.e. block until the ack is received, or return - * after the hello is sent and call a connection callback when the hello - * ack is received). - */ - -//! \internal -typedef struct pcmk_controld_api_s pcmk_controld_api_t; - -//! \internal -typedef struct pcmk_controld_api_callback_s { - void (*callback)(pcmk_controld_api_t *api, void *api_data, void *user_data); - void *user_data; -} pcmk_controld_api_cb_t; - -//! \internal -struct pcmk_controld_api_s { - //! \internal - void *private; - - /*! - * \internal - * \brief Connect to the local controller - * - * \param[in] api Controller API instance - * \param[in] use_mainloop If true, attach IPC to main loop - * \param[in] dispatch_cb If not NULL, call this when replies are received - * \param[in] destroy_cb If not NULL, call this if connection drops - * - * \return Standard Pacemaker return code - * \note Only the pointers inside the callback objects need to be - * persistent, not the callback objects themselves. The destroy_cb - * will be called only for unrequested disconnects. - */ - int (*connect)(pcmk_controld_api_t *api, bool use_mainloop, - pcmk_controld_api_cb_t *dispatch_cb, - pcmk_controld_api_cb_t *destroy_cb); - - /*! - * \internal - * \brief Disconnect from the local controller - * - * \param[in] api Controller API instance - * - * \return Standard Pacemaker return code - */ - int (*disconnect)(pcmk_controld_api_t *api); - - /*! - * \internal - * \brief Check number of replies still expected from controller - * - * \param[in] api Controller API instance - * - * \return Number of expected replies - */ - unsigned int (*replies_expected)(pcmk_controld_api_t *api); - - /*! - * \internal - * \brief Send a reprobe controller operation - * - * \param[in] api Controller API instance - * \param[in] target_node Name of node to reprobe - * \param[in] router_node Router node for host - * - * \return Standard Pacemaker return code - */ - int (*reprobe)(pcmk_controld_api_t *api, const char *target_node, - const char *router_node); - - /* @TODO These methods have a lot of arguments. One possibility would be to - * make a struct for agent info (standard/provider/type), which theortically - * could be used throughout pacemaker code. However that would end up being - * really awkward to use generically, since sometimes you need to allocate - * those strings (char *) and other times you only have references into XML - * (const char *). We could make some structs just for this API. - */ - - /*! - * \internal - * \brief Ask the controller to fail a resource - * - * \param[in] api Controller API instance - * \param[in] target_node Name of node resource is on - * \param[in] router_node Router node for target - * \param[in] rsc_id ID of resource to fail - * \param[in] rsc_long_id Long ID of resource (if any) - * \param[in] standard Standard of resource - * \param[in] provider Provider of resource (if any) - * \param[in] type Type of resource to fail - * - * \return Standard Pacemaker return code - */ - int (*fail_resource)(pcmk_controld_api_t *api, const char *target_node, - const char *router_node, const char *rsc_id, - const char *rsc_long_id, const char *standard, - const char *provider, const char *type); - - /*! - * \internal - * \brief Ask the controller to refresh a resource - * - * \param[in] api Controller API instance - * \param[in] target_node Name of node resource is on - * \param[in] router_node Router node for target - * \param[in] rsc_id ID of resource to refresh - * \param[in] rsc_long_id Long ID of resource (if any) - * \param[in] standard Standard of resource - * \param[in] provider Provider of resource (if any) - * \param[in] type Type of resource - * \param[in] cib_only If true, clean resource from CIB only - * - * \return Standard Pacemaker return code - */ - int (*refresh_resource)(pcmk_controld_api_t *api, const char *target_node, - const char *router_node, const char *rsc_id, - const char *rsc_long_id, const char *standard, - const char *provider, const char *type, - bool cib_only); -}; - -/*! - * \internal - * \brief Create new controller IPC API object for clients - * - * \param[in] client_name Client name to use with IPC - * \param[in] client_uuid Client UUID to use with IPC - * - * \return Newly allocated object - * \note This function asserts on errors, so it will never return NULL. - * The caller is responsible for freeing the result with - * pcmk_free_controld_api(). - */ -pcmk_controld_api_t *pcmk_new_controld_api(const char *client_name, - const char *client_uuid); - -/*! - * \internal - * \brief Free a controller IPC API object - * - * \param[in] api Controller IPC API object to free - */ -void pcmk_free_controld_api(pcmk_controld_api_t *api); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c index 37789d1..cc2abeb 100644 --- a/tools/crm_resource_runtime.c +++ b/tools/crm_resource_runtime.c @@ -8,6 +8,7 @@ */ #include +#include int resource_verbose = 0; bool do_force = FALSE; @@ -460,7 +461,7 @@ cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name, // \return Standard Pacemaker return code static int -send_lrm_rsc_op(pcmk_controld_api_t *controld_api, bool do_fail_resource, +send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource, const char *host_uname, const char *rsc_id, pe_working_set_t *data_set) { @@ -528,14 +529,13 @@ send_lrm_rsc_op(pcmk_controld_api_t *controld_api, bool do_fail_resource, rsc_api_id = rsc->id; } if (do_fail_resource) { - return controld_api->fail_resource(controld_api, host_uname, - router_node, rsc_api_id, rsc_long_id, - rsc_class, rsc_provider, rsc_type); + return pcmk_controld_api_fail(controld_api, host_uname, router_node, + rsc_api_id, rsc_long_id, + rsc_class, rsc_provider, rsc_type); } else { - return controld_api->refresh_resource(controld_api, host_uname, - router_node, rsc_api_id, - rsc_long_id, rsc_class, - rsc_provider, rsc_type, cib_only); + return pcmk_controld_api_refresh(controld_api, host_uname, router_node, + rsc_api_id, rsc_long_id, rsc_class, + rsc_provider, rsc_type, cib_only); } } @@ -558,7 +558,7 @@ rsc_fail_name(pe_resource_t *rsc) // \return Standard Pacemaker return code static int -clear_rsc_history(pcmk_controld_api_t *controld_api, const char *host_uname, +clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname, const char *rsc_id, pe_working_set_t *data_set) { int rc = pcmk_ok; @@ -574,16 +574,16 @@ clear_rsc_history(pcmk_controld_api_t *controld_api, const char *host_uname, } crm_trace("Processing %d mainloop inputs", - controld_api->replies_expected(controld_api)); + pcmk_controld_api_replies_expected(controld_api)); while (g_main_context_iteration(NULL, FALSE)) { crm_trace("Processed mainloop input, %d still remaining", - controld_api->replies_expected(controld_api)); + pcmk_controld_api_replies_expected(controld_api)); } return rc; } static int -clear_rsc_failures(pcmk_controld_api_t *controld_api, const char *node_name, +clear_rsc_failures(pcmk_ipc_api_t *controld_api, const char *node_name, const char *rsc_id, const char *operation, const char *interval_spec, pe_working_set_t *data_set) { @@ -683,7 +683,7 @@ clear_rsc_fail_attrs(pe_resource_t *rsc, const char *operation, } int -cli_resource_delete(pcmk_controld_api_t *controld_api, const char *host_uname, +cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname, pe_resource_t *rsc, const char *operation, const char *interval_spec, bool just_failures, pe_working_set_t *data_set) @@ -792,7 +792,7 @@ cli_resource_delete(pcmk_controld_api_t *controld_api, const char *host_uname, } int -cli_cleanup_all(pcmk_controld_api_t *controld_api, const char *node_name, +cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name, const char *operation, const char *interval_spec, pe_working_set_t *data_set) { @@ -905,7 +905,7 @@ cli_resource_check(cib_t * cib_conn, pe_resource_t *rsc) // \return Standard Pacemaker return code int -cli_resource_fail(pcmk_controld_api_t *controld_api, const char *host_uname, +cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname, const char *rsc_id, pe_working_set_t *data_set) { crm_notice("Failing %s on %s", rsc_id, host_uname); -- 1.8.3.1 From ae14fa4a831e45eae0d78b0f42765fcf40c4ce56 Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 14 Apr 2020 14:06:02 -0500 Subject: [PATCH 4/6] Refactor: tools: convert crm_node to use new controller IPC model --- tools/crm_node.c | 281 +++++++++++++++++++++++++++---------------------------- 1 file changed, 140 insertions(+), 141 deletions(-) diff --git a/tools/crm_node.c b/tools/crm_node.c index db31f20..1773a36 100644 --- a/tools/crm_node.c +++ b/tools/crm_node.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #define SUMMARY "crm_node - Tool for displaying low-level node information" @@ -39,7 +40,6 @@ gboolean command_cb(const gchar *option_name, const gchar *optarg, gpointer data gboolean name_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error); gboolean remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error); -static char *pid_s = NULL; static GMainLoop *mainloop = NULL; static crm_exit_t exit_code = CRM_EX_OK; @@ -140,11 +140,6 @@ remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError * static void crm_node_exit(crm_exit_t value) { - if (pid_s) { - free(pid_s); - pid_s = NULL; - } - exit_code = value; if (mainloop && g_main_loop_is_running(mainloop)) { @@ -155,173 +150,123 @@ crm_node_exit(crm_exit_t value) } static void -exit_disconnect(gpointer user_data) -{ - fprintf(stderr, "error: Lost connection to cluster\n"); - crm_node_exit(CRM_EX_DISCONNECT); -} - -typedef int (*ipc_dispatch_fn) (const char *buffer, ssize_t length, - gpointer userdata); - -static crm_ipc_t * -new_mainloop_for_ipc(const char *system, ipc_dispatch_fn dispatch) +controller_event_cb(pcmk_ipc_api_t *controld_api, + enum pcmk_ipc_event event_type, crm_exit_t status, + void *event_data, void *user_data) { - mainloop_io_t *source = NULL; - crm_ipc_t *ipc = NULL; - - struct ipc_client_callbacks ipc_callbacks = { - .dispatch = dispatch, - .destroy = exit_disconnect - }; + pcmk_controld_api_reply_t *reply = event_data; - mainloop = g_main_loop_new(NULL, FALSE); - source = mainloop_add_ipc_client(system, G_PRIORITY_DEFAULT, 0, - NULL, &ipc_callbacks); - ipc = mainloop_get_ipc_client(source); - if (ipc == NULL) { - fprintf(stderr, - "error: Could not connect to cluster (is it running?)\n"); - crm_node_exit(CRM_EX_DISCONNECT); - } - return ipc; -} - -static int -send_controller_hello(crm_ipc_t *controller) -{ - xmlNode *hello = NULL; - int rc; - - pid_s = pcmk__getpid_s(); - hello = create_hello_message(pid_s, crm_system_name, "1", "0"); - rc = crm_ipc_send(controller, hello, 0, 0, NULL); - free_xml(hello); - return (rc < 0)? rc : 0; -} - -static int -send_node_info_request(crm_ipc_t *controller, uint32_t nodeid) -{ - xmlNode *ping = NULL; - int rc; - - ping = create_request(CRM_OP_NODE_INFO, NULL, NULL, CRM_SYSTEM_CRMD, - crm_system_name, pid_s); - if (nodeid > 0) { - crm_xml_add_int(ping, XML_ATTR_ID, nodeid); - } - rc = crm_ipc_send(controller, ping, 0, 0, NULL); - free_xml(ping); - return (rc < 0)? rc : 0; -} + switch (event_type) { + case pcmk_ipc_event_disconnect: + if (exit_code == CRM_EX_DISCONNECT) { // Unexpected + fprintf(stderr, "error: Lost connection to controller\n"); + } + goto done; + break; -static int -dispatch_controller(const char *buffer, ssize_t length, gpointer userdata) -{ - xmlNode *message = string2xml(buffer); - xmlNode *data = NULL; - const char *value = NULL; + case pcmk_ipc_event_reply: + break; - if (message == NULL) { - fprintf(stderr, "error: Could not understand reply from controller\n"); - crm_node_exit(CRM_EX_PROTOCOL); - return 0; + default: + return; } - crm_log_xml_trace(message, "controller reply"); - - exit_code = CRM_EX_PROTOCOL; - // Validate reply - value = crm_element_value(message, F_CRM_MSG_TYPE); - if (safe_str_neq(value, XML_ATTR_RESPONSE)) { - fprintf(stderr, "error: Message from controller was not a reply\n"); + if (status != CRM_EX_OK) { + fprintf(stderr, "error: Bad reply from controller: %s\n", + crm_exit_str(status)); goto done; } - value = crm_element_value(message, XML_ATTR_REFERENCE); - if (value == NULL) { - fprintf(stderr, "error: Controller reply did not specify original message\n"); - goto done; - } - data = get_message_xml(message, F_CRM_DATA); - if (data == NULL) { - fprintf(stderr, "error: Controller reply did not contain any data\n"); + if (reply->reply_type != pcmk_controld_reply_info) { + fprintf(stderr, "error: Unknown reply type %d from controller\n", + reply->reply_type); goto done; } + // Parse desired info from reply and display to user switch (options.command) { case 'i': - value = crm_element_value(data, XML_ATTR_ID); - if (value == NULL) { - fprintf(stderr, "error: Controller reply did not contain node ID\n"); - } else { - printf("%s\n", value); - exit_code = CRM_EX_OK; + if (reply->data.node_info.id == 0) { + fprintf(stderr, + "error: Controller reply did not contain node ID\n"); + exit_code = CRM_EX_PROTOCOL; + goto done; } + printf("%d\n", reply->data.node_info.id); break; case 'n': case 'N': - value = crm_element_value(data, XML_ATTR_UNAME); - if (value == NULL) { + if (reply->data.node_info.uname == NULL) { fprintf(stderr, "Node is not known to cluster\n"); exit_code = CRM_EX_NOHOST; - } else { - printf("%s\n", value); - exit_code = CRM_EX_OK; + goto done; } + printf("%s\n", reply->data.node_info.uname); break; case 'q': - value = crm_element_value(data, XML_ATTR_HAVE_QUORUM); - if (value == NULL) { - fprintf(stderr, "error: Controller reply did not contain quorum status\n"); - } else { - bool quorum = crm_is_true(value); - - printf("%d\n", quorum); - exit_code = quorum? CRM_EX_OK : CRM_EX_QUORUM; + printf("%d\n", reply->data.node_info.have_quorum); + if (!(reply->data.node_info.have_quorum)) { + exit_code = CRM_EX_QUORUM; + goto done; } break; default: fprintf(stderr, "internal error: Controller reply not expected\n"); exit_code = CRM_EX_SOFTWARE; - break; + goto done; } + // Success + exit_code = CRM_EX_OK; done: - free_xml(message); - crm_node_exit(exit_code); - return 0; + pcmk_disconnect_ipc(controld_api); + pcmk_quit_main_loop(mainloop, 10); } static void run_controller_mainloop(uint32_t nodeid) { - crm_ipc_t *controller = NULL; + pcmk_ipc_api_t *controld_api = NULL; int rc; - controller = new_mainloop_for_ipc(CRM_SYSTEM_CRMD, dispatch_controller); + // Set disconnect exit code to handle unexpected disconnects + exit_code = CRM_EX_DISCONNECT; + + // Create controller IPC object + rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld); + if (rc != pcmk_rc_ok) { + fprintf(stderr, "error: Could not connect to controller: %s\n", + pcmk_rc_str(rc)); + return; + } + pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL); - rc = send_controller_hello(controller); - if (rc < 0) { - fprintf(stderr, "error: Could not register with controller: %s\n", - pcmk_strerror(rc)); - crm_node_exit(crm_errno2exit(rc)); + // Connect to controller + rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main); + if (rc != pcmk_rc_ok) { + fprintf(stderr, "error: Could not connect to controller: %s\n", + pcmk_rc_str(rc)); + exit_code = pcmk_rc2exitc(rc); + return; } - rc = send_node_info_request(controller, nodeid); - if (rc < 0) { + rc = pcmk_controld_api_node_info(controld_api, nodeid); + if (rc != pcmk_rc_ok) { fprintf(stderr, "error: Could not ping controller: %s\n", - pcmk_strerror(rc)); - crm_node_exit(crm_errno2exit(rc)); + pcmk_rc_str(rc)); + pcmk_disconnect_ipc(controld_api); + exit_code = pcmk_rc2exitc(rc); + return; } - // Run main loop to get controller reply via dispatch_controller() + // Run main loop to get controller reply via controller_event_cb() + mainloop = g_main_loop_new(NULL, FALSE); g_main_loop_run(mainloop); g_main_loop_unref(mainloop); mainloop = NULL; + pcmk_free_ipc_api(controld_api); } static void @@ -385,32 +330,56 @@ cib_remove_node(long id, const char *name) } static int +controller_remove_node(const char *node_name, long nodeid) +{ + pcmk_ipc_api_t *controld_api = NULL; + int rc; + + // Create controller IPC object + rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld); + if (rc != pcmk_rc_ok) { + fprintf(stderr, "error: Could not connect to controller: %s\n", + pcmk_rc_str(rc)); + return ENOTCONN; + } + + // Connect to controller (without main loop) + rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_sync); + if (rc != pcmk_rc_ok) { + fprintf(stderr, "error: Could not connect to controller: %s\n", + pcmk_rc_str(rc)); + pcmk_free_ipc_api(controld_api); + return rc; + } + + rc = pcmk_ipc_purge_node(controld_api, node_name, nodeid); + if (rc != pcmk_rc_ok) { + fprintf(stderr, + "error: Could not clear node from controller's cache: %s\n", + pcmk_rc_str(rc)); + } + + pcmk_free_ipc_api(controld_api); + return pcmk_rc_ok; +} + +static int tools_remove_node_cache(const char *node_name, long nodeid, const char *target) { int rc = -1; - crm_ipc_t *conn = crm_ipc_new(target, 0); + crm_ipc_t *conn = NULL; xmlNode *cmd = NULL; + conn = crm_ipc_new(target, 0); if (!conn) { return -ENOTCONN; } - if (!crm_ipc_connect(conn)) { crm_perror(LOG_ERR, "Connection to %s failed", target); crm_ipc_destroy(conn); return -ENOTCONN; } - if(safe_str_eq(target, CRM_SYSTEM_CRMD)) { - // The controller requires a hello message before sending a request - rc = send_controller_hello(conn); - if (rc < 0) { - fprintf(stderr, "error: Could not register with controller: %s\n", - pcmk_strerror(rc)); - return rc; - } - } - crm_trace("Removing %s[%ld] from the %s membership cache", node_name, nodeid, target); @@ -427,9 +396,9 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target) crm_xml_add_int(cmd, PCMK__XA_ATTR_NODE_ID, (int) nodeid); } - } else { - cmd = create_request(CRM_OP_RM_NODE_CACHE, - NULL, NULL, target, crm_system_name, pid_s); + } else { // Fencer or pacemakerd + cmd = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, target, + crm_system_name, NULL); if (nodeid > 0) { crm_xml_set_id(cmd, "%ld", nodeid); } @@ -441,6 +410,7 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target) target, node_name, nodeid, rc); if (rc > 0) { + // @TODO Should this be done just once after all the rest? rc = cib_remove_node(nodeid, node_name); } @@ -455,12 +425,12 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target) static void remove_node(const char *target_uname) { + int rc; int d = 0; long nodeid = 0; const char *node_name = NULL; char *endptr = NULL; const char *daemons[] = { - CRM_SYSTEM_CRMD, "stonith-ng", T_ATTRD, CRM_SYSTEM_MCP, @@ -476,6 +446,12 @@ remove_node(const char *target_uname) node_name = target_uname; } + rc = controller_remove_node(node_name, nodeid); + if (rc != pcmk_rc_ok) { + exit_code = pcmk_rc2exitc(rc); + return; + } + for (d = 0; d < DIMOF(daemons); d++) { if (tools_remove_node_cache(node_name, nodeid, daemons[d])) { crm_err("Failed to connect to %s to remove node '%s'", @@ -545,12 +521,34 @@ node_mcp_dispatch(const char *buffer, ssize_t length, gpointer userdata) } static void +lost_pacemakerd(gpointer user_data) +{ + fprintf(stderr, "error: Lost connection to cluster\n"); + exit_code = CRM_EX_DISCONNECT; + g_main_loop_quit(mainloop); +} + +static void run_pacemakerd_mainloop(void) { crm_ipc_t *ipc = NULL; xmlNode *poke = NULL; + mainloop_io_t *source = NULL; - ipc = new_mainloop_for_ipc(CRM_SYSTEM_MCP, node_mcp_dispatch); + struct ipc_client_callbacks ipc_callbacks = { + .dispatch = node_mcp_dispatch, + .destroy = lost_pacemakerd + }; + + source = mainloop_add_ipc_client(CRM_SYSTEM_MCP, G_PRIORITY_DEFAULT, 0, + NULL, &ipc_callbacks); + ipc = mainloop_get_ipc_client(source); + if (ipc == NULL) { + fprintf(stderr, + "error: Could not connect to cluster (is it running?)\n"); + exit_code = CRM_EX_DISCONNECT; + return; + } // Sending anything will get us a list of nodes poke = create_xml_node(NULL, "poke"); @@ -558,6 +556,7 @@ run_pacemakerd_mainloop(void) free_xml(poke); // Handle reply via node_mcp_dispatch() + mainloop = g_main_loop_new(NULL, FALSE); g_main_loop_run(mainloop); g_main_loop_unref(mainloop); mainloop = NULL; -- 1.8.3.1 From a361f764cb28630d440eec0f3e04a4f3812825eb Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Tue, 14 Apr 2020 16:05:15 -0500 Subject: [PATCH 5/6] Refactor: tools: remove dead code from crm_node --- tools/crm_node.c | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/tools/crm_node.c b/tools/crm_node.c index 1773a36..57c2ee5 100644 --- a/tools/crm_node.c +++ b/tools/crm_node.c @@ -130,25 +130,6 @@ remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError * return TRUE; } -/*! - * \internal - * \brief Exit crm_node - * Clean up memory, and either quit mainloop (if running) or exit - * - * \param[in] value Exit status - */ -static void -crm_node_exit(crm_exit_t value) -{ - exit_code = value; - - if (mainloop && g_main_loop_is_running(mainloop)) { - g_main_loop_quit(mainloop); - } else { - crm_exit(exit_code); - } -} - static void controller_event_cb(pcmk_ipc_api_t *controld_api, enum pcmk_ipc_event event_type, crm_exit_t status, @@ -660,6 +641,5 @@ done: g_strfreev(processed_args); g_clear_error(&error); pcmk__free_arg_context(context); - crm_node_exit(exit_code); - return exit_code; + return crm_exit(exit_code); } -- 1.8.3.1 From 591944539259f6294450517770d95c7a02fc599c Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Mon, 20 Apr 2020 15:48:15 -0500 Subject: [PATCH 6/6] Refactor: tools: convert crmadmin to use new controller IPC model --- tools/crmadmin.c | 484 ++++++++++++++++++++++++------------------------------- 1 file changed, 208 insertions(+), 276 deletions(-) diff --git a/tools/crmadmin.c b/tools/crmadmin.c index 3e9e959..4688458 100644 --- a/tools/crmadmin.c +++ b/tools/crmadmin.c @@ -9,56 +9,44 @@ #include -#include - #include -#include -#include +#include +#include // atoi() -#include -#include -#include -#include #include // gboolean, GMainLoop, etc. +#include // xmlNode #include +#include #include #include - +#include #include -#include - #define DEFAULT_MESSAGE_TIMEOUT_MS 30000 static guint message_timer_id = 0; static guint message_timeout_ms = DEFAULT_MESSAGE_TIMEOUT_MS; static GMainLoop *mainloop = NULL; -static crm_ipc_t *crmd_channel = NULL; -static char *admin_uuid = NULL; - -gboolean do_init(void); -int do_work(void); -void crmadmin_ipc_connection_destroy(gpointer user_data); -int admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata); -int do_find_node_list(xmlNode * xml_node); + +bool do_work(pcmk_ipc_api_t *api); +void do_find_node_list(xmlNode *xml_node); gboolean admin_message_timeout(gpointer data); +static enum { + cmd_none, + cmd_shutdown, + cmd_health, + cmd_elect_dc, + cmd_whois_dc, + cmd_list_nodes, +} command = cmd_none; + static gboolean BE_VERBOSE = FALSE; -static int expected_responses = 1; static gboolean BASH_EXPORT = FALSE; -static gboolean DO_HEALTH = FALSE; -static gboolean DO_RESET = FALSE; -static gboolean DO_RESOURCE = FALSE; -static gboolean DO_ELECT_DC = FALSE; -static gboolean DO_WHOIS_DC = FALSE; -static gboolean DO_NODE_LIST = FALSE; static gboolean BE_SILENT = FALSE; -static gboolean DO_RESOURCE_LIST = FALSE; -static const char *crmd_operation = NULL; static char *dest_node = NULL; static crm_exit_t exit_code = CRM_EX_OK; -static const char *sys_to = NULL; static pcmk__cli_option_t long_options[] = { // long option, argument type, storage, short option, description, flags @@ -133,7 +121,8 @@ static pcmk__cli_option_t long_options[] = { }, { "bash-export", no_argument, NULL, 'B', - "Create Bash export entries of the form 'export uname=uuid'\n", + "Display nodes as shell commands of the form 'export uname=uuid' " + "(valid with -N/--nodes)'\n", pcmk__option_default }, { @@ -142,13 +131,98 @@ static pcmk__cli_option_t long_options[] = { }, { "-spacer-", no_argument, NULL, '-', - " The -K and -E commands are rarely used and may be removed in " - "future versions.", + "The -K and -E commands do not work and may be removed in a future " + "version.", pcmk__option_default }, { 0, 0, 0, 0 } }; +static void +quit_main_loop(crm_exit_t ec) +{ + exit_code = ec; + if (mainloop != NULL) { + GMainLoop *mloop = mainloop; + + mainloop = NULL; // Don't re-enter this block + pcmk_quit_main_loop(mloop, 10); + g_main_loop_unref(mloop); + } +} + +static void +controller_event_cb(pcmk_ipc_api_t *controld_api, + enum pcmk_ipc_event event_type, crm_exit_t status, + void *event_data, void *user_data) +{ + pcmk_controld_api_reply_t *reply = event_data; + + switch (event_type) { + case pcmk_ipc_event_disconnect: + if (exit_code == CRM_EX_DISCONNECT) { // Unexpected + fprintf(stderr, "error: Lost connection to controller\n"); + } + goto done; + break; + + case pcmk_ipc_event_reply: + break; + + default: + return; + } + + if (message_timer_id != 0) { + g_source_remove(message_timer_id); + message_timer_id = 0; + } + + if (status != CRM_EX_OK) { + fprintf(stderr, "error: Bad reply from controller: %s", + crm_exit_str(status)); + exit_code = status; + goto done; + } + + if (reply->reply_type != pcmk_controld_reply_ping) { + fprintf(stderr, "error: Unknown reply type %d from controller\n", + reply->reply_type); + goto done; + } + + // Parse desired information from reply + switch (command) { + case cmd_health: + printf("Status of %s@%s: %s (%s)\n", + reply->data.ping.sys_from, + reply->host_from, + reply->data.ping.fsa_state, + reply->data.ping.result); + if (BE_SILENT && (reply->data.ping.fsa_state != NULL)) { + fprintf(stderr, "%s\n", reply->data.ping.fsa_state); + } + exit_code = CRM_EX_OK; + break; + + case cmd_whois_dc: + printf("Designated Controller is: %s\n", reply->host_from); + if (BE_SILENT && (reply->host_from != NULL)) { + fprintf(stderr, "%s\n", reply->host_from); + } + exit_code = CRM_EX_OK; + break; + + default: // Not really possible here + exit_code = CRM_EX_SOFTWARE; + break; + } + +done: + pcmk_disconnect_ipc(controld_api); + quit_main_loop(exit_code); +} + // \return Standard Pacemaker return code static int list_nodes() @@ -181,6 +255,9 @@ main(int argc, char **argv) int option_index = 0; int argerr = 0; int flag; + int rc; + pcmk_ipc_api_t *controld_api = NULL; + bool need_controld_api = true; crm_log_cli_init("crmadmin"); pcmk__set_cli_options(NULL, " [options]", long_options, @@ -211,33 +288,40 @@ main(int argc, char **argv) pcmk__cli_help(flag, CRM_EX_OK); break; case 'D': - DO_WHOIS_DC = TRUE; + command = cmd_whois_dc; break; case 'B': BASH_EXPORT = TRUE; break; case 'K': - DO_RESET = TRUE; + command = cmd_shutdown; crm_trace("Option %c => %s", flag, optarg); + if (dest_node != NULL) { + free(dest_node); + } dest_node = strdup(optarg); - crmd_operation = CRM_OP_LOCAL_SHUTDOWN; break; case 'q': BE_SILENT = TRUE; break; case 'S': - DO_HEALTH = TRUE; + command = cmd_health; crm_trace("Option %c => %s", flag, optarg); + if (dest_node != NULL) { + free(dest_node); + } dest_node = strdup(optarg); break; case 'E': - DO_ELECT_DC = TRUE; + command = cmd_elect_dc; break; case 'N': - DO_NODE_LIST = TRUE; + command = cmd_list_nodes; + need_controld_api = false; break; case 'H': - DO_HEALTH = TRUE; + fprintf(stderr, "Cluster-wide health option not supported\n"); + ++argerr; break; default: printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag); @@ -257,285 +341,133 @@ main(int argc, char **argv) ++argerr; } + if (command == cmd_none) { + fprintf(stderr, "error: Must specify a command option\n\n"); + ++argerr; + } + if (argerr) { pcmk__cli_help('?', CRM_EX_USAGE); } - if (do_init()) { - int res = 0; - - res = do_work(); - if (res > 0) { - /* wait for the reply by creating a mainloop and running it until - * the callbacks are invoked... - */ - mainloop = g_main_loop_new(NULL, FALSE); - crm_trace("Waiting for %d replies from the local CRM", expected_responses); - - message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL); - - g_main_loop_run(mainloop); - - } else if (res < 0) { - crm_err("No message to send"); - exit_code = CRM_EX_ERROR; + // Connect to the controller if needed + if (need_controld_api) { + rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld); + if (controld_api == NULL) { + fprintf(stderr, "error: Could not connect to controller: %s\n", + pcmk_rc_str(rc)); + exit_code = pcmk_rc2exitc(rc); + goto done; } - } else { - crm_warn("Init failed, could not perform requested operations"); - exit_code = CRM_EX_UNAVAILABLE; - } - - crm_trace("%s exiting normally", crm_system_name); - return exit_code; -} - -int -do_work(void) -{ - int ret = 1; - - /* construct the request */ - xmlNode *msg_data = NULL; - gboolean all_is_good = TRUE; - - if (DO_HEALTH == TRUE) { - crm_trace("Querying the system"); - - sys_to = CRM_SYSTEM_DC; - - if (dest_node != NULL) { - sys_to = CRM_SYSTEM_CRMD; - crmd_operation = CRM_OP_PING; - - if (BE_VERBOSE) { - expected_responses = 1; - } - - } else { - crm_info("Cluster-wide health not available yet"); - all_is_good = FALSE; + pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL); + rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main); + if (rc != pcmk_rc_ok) { + fprintf(stderr, "error: Could not connect to controller: %s\n", + pcmk_rc_str(rc)); + exit_code = pcmk_rc2exitc(rc); + goto done; } - - } else if (DO_ELECT_DC) { - /* tell the local node to initiate an election */ - - dest_node = NULL; - sys_to = CRM_SYSTEM_CRMD; - crmd_operation = CRM_OP_VOTE; - ret = 0; /* no return message */ - - } else if (DO_WHOIS_DC) { - dest_node = NULL; - sys_to = CRM_SYSTEM_DC; - crmd_operation = CRM_OP_PING; - - } else if (DO_NODE_LIST) { - crm_exit(pcmk_rc2exitc(list_nodes())); - - } else if (DO_RESET) { - /* tell dest_node to initiate the shutdown procedure - * - * if dest_node is NULL, the request will be sent to the - * local node - */ - sys_to = CRM_SYSTEM_CRMD; - ret = 0; /* no return message */ - - } else { - crm_err("Unknown options"); - all_is_good = FALSE; } - if (all_is_good == FALSE) { - crm_err("Creation of request failed. No message to send"); - return -1; + if (do_work(controld_api)) { + // A reply is needed from controller, so run main loop to get it + exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects + mainloop = g_main_loop_new(NULL, FALSE); + message_timer_id = g_timeout_add(message_timeout_ms, + admin_message_timeout, NULL); + g_main_loop_run(mainloop); } -/* send it */ - if (crmd_channel == NULL) { - crm_err("The IPC connection is not valid, cannot send anything"); - return -1; - } - - if (sys_to == NULL) { - if (dest_node != NULL) { - sys_to = CRM_SYSTEM_CRMD; - } else { - sys_to = CRM_SYSTEM_DC; - } - } - - { - xmlNode *cmd = create_request(crmd_operation, msg_data, dest_node, sys_to, - crm_system_name, admin_uuid); - - crm_ipc_send(crmd_channel, cmd, 0, 0, NULL); - free_xml(cmd); - } - - return ret; -} +done: + if (controld_api != NULL) { + pcmk_ipc_api_t *capi = controld_api; -void -crmadmin_ipc_connection_destroy(gpointer user_data) -{ - crm_err("Connection to controller was terminated"); - if (mainloop) { - g_main_loop_quit(mainloop); - } else { - crm_exit(CRM_EX_DISCONNECT); + controld_api = NULL; // Ensure we can't free this twice + pcmk_free_ipc_api(capi); } -} - -struct ipc_client_callbacks crm_callbacks = { - .dispatch = admin_msg_callback, - .destroy = crmadmin_ipc_connection_destroy -}; - -gboolean -do_init(void) -{ - mainloop_io_t *source = - mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks); - - admin_uuid = pcmk__getpid_s(); - - crmd_channel = mainloop_get_ipc_client(source); - - if (DO_RESOURCE || DO_RESOURCE_LIST || DO_NODE_LIST) { - return TRUE; - - } else if (crmd_channel != NULL) { - xmlNode *xml = create_hello_message(admin_uuid, crm_system_name, "0", "1"); - - crm_ipc_send(crmd_channel, xml, 0, 0, NULL); - return TRUE; + if (mainloop != NULL) { + g_main_loop_unref(mainloop); + mainloop = NULL; } - return FALSE; + return crm_exit(exit_code); } -static bool -validate_crm_message(xmlNode * msg, const char *sys, const char *uuid, const char *msg_type) +// \return True if reply from controller is needed +bool +do_work(pcmk_ipc_api_t *controld_api) { - const char *type = NULL; - const char *crm_msg_reference = NULL; - - if (msg == NULL) { - return FALSE; - } + bool need_reply = false; + int rc = pcmk_rc_ok; - type = crm_element_value(msg, F_CRM_MSG_TYPE); - crm_msg_reference = crm_element_value(msg, XML_ATTR_REFERENCE); - - if (type == NULL) { - crm_info("No message type defined."); - return FALSE; - - } else if (msg_type != NULL && strcasecmp(msg_type, type) != 0) { - crm_info("Expecting a (%s) message but received a (%s).", msg_type, type); - return FALSE; - } - - if (crm_msg_reference == NULL) { - crm_info("No message crm_msg_reference defined."); - return FALSE; - } - - return TRUE; -} - -int -admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata) -{ - static int received_responses = 0; - xmlNode *xml = string2xml(buffer); - - received_responses++; - g_source_remove(message_timer_id); - - crm_log_xml_trace(xml, "ipc"); - - if (xml == NULL) { - crm_info("XML in IPC message was not valid... " "discarding."); - - } else if (validate_crm_message(xml, crm_system_name, admin_uuid, XML_ATTR_RESPONSE) == FALSE) { - crm_trace("Message was not a CRM response. Discarding."); - - } else if (DO_HEALTH) { - xmlNode *data = get_message_xml(xml, F_CRM_DATA); - const char *state = crm_element_value(data, XML_PING_ATTR_CRMDSTATE); + switch (command) { + case cmd_shutdown: + rc = pcmk_controld_api_shutdown(controld_api, dest_node); + break; - printf("Status of %s@%s: %s (%s)\n", - crm_element_value(data, XML_PING_ATTR_SYSFROM), - crm_element_value(xml, F_CRM_HOST_FROM), - state, crm_element_value(data, XML_PING_ATTR_STATUS)); + case cmd_health: // dest_node != NULL + case cmd_whois_dc: // dest_node == NULL + rc = pcmk_controld_api_ping(controld_api, dest_node); + need_reply = true; + break; - if (BE_SILENT && state != NULL) { - fprintf(stderr, "%s\n", state); - } + case cmd_elect_dc: + rc = pcmk_controld_api_start_election(controld_api); + break; - } else if (DO_WHOIS_DC) { - const char *dc = crm_element_value(xml, F_CRM_HOST_FROM); + case cmd_list_nodes: + rc = list_nodes(); + break; - printf("Designated Controller is: %s\n", dc); - if (BE_SILENT && dc != NULL) { - fprintf(stderr, "%s\n", dc); - } - crm_exit(CRM_EX_OK); + case cmd_none: // not actually possible here + break; } - - free_xml(xml); - - if (received_responses >= expected_responses) { - crm_trace("Received expected number (%d) of replies, exiting normally", - expected_responses); - crm_exit(CRM_EX_OK); + if (rc != pcmk_rc_ok) { + fprintf(stderr, "error: Command failed: %s", pcmk_rc_str(rc)); + exit_code = pcmk_rc2exitc(rc); } - - message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL); - return 0; + return need_reply; } gboolean admin_message_timeout(gpointer data) { - fprintf(stderr, "No messages received in %d seconds.. aborting\n", - (int)message_timeout_ms / 1000); - crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000); - exit_code = CRM_EX_TIMEOUT; - g_main_loop_quit(mainloop); - return FALSE; + fprintf(stderr, + "error: No reply received from controller before timeout (%dms)\n", + message_timeout_ms); + message_timer_id = 0; + quit_main_loop(CRM_EX_TIMEOUT); + return FALSE; // Tells glib to remove source } -int +void do_find_node_list(xmlNode * xml_node) { int found = 0; xmlNode *node = NULL; xmlNode *nodes = get_object_root(XML_CIB_TAG_NODES, xml_node); - for (node = __xml_first_child_element(nodes); node != NULL; - node = __xml_next_element(node)) { + for (node = first_named_child(nodes, XML_CIB_TAG_NODE); node != NULL; + node = crm_next_same_xml(node)) { - if (crm_str_eq((const char *)node->name, XML_CIB_TAG_NODE, TRUE)) { + if (BASH_EXPORT) { + printf("export %s=%s\n", + crm_element_value(node, XML_ATTR_UNAME), + crm_element_value(node, XML_ATTR_ID)); + } else { + const char *node_type = crm_element_value(node, XML_ATTR_TYPE); - if (BASH_EXPORT) { - printf("export %s=%s\n", - crm_element_value(node, XML_ATTR_UNAME), - crm_element_value(node, XML_ATTR_ID)); - } else { - printf("%s node: %s (%s)\n", - crm_element_value(node, XML_ATTR_TYPE), - crm_element_value(node, XML_ATTR_UNAME), - crm_element_value(node, XML_ATTR_ID)); + if (node_type == NULL) { + node_type = "member"; } - found++; + printf("%s node: %s (%s)\n", node_type, + crm_element_value(node, XML_ATTR_UNAME), + crm_element_value(node, XML_ATTR_ID)); } + found++; } + // @TODO List Pacemaker Remote nodes that don't have a entry if (found == 0) { - printf("NO nodes configured\n"); + printf("No nodes configured\n"); } - - return found; } -- 1.8.3.1