From e01a1178fd8f5b99895683b3af9998e9485d12a4 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Fri, 24 Apr 2020 16:42:02 -0500
Subject: [PATCH 1/3] Feature: controller: add new IPC API command for getting
node list
This is based on and will replace the corresponding functionality in
pacemakerd.
---
daemons/controld/controld_messages.c | 44 ++++++++++++++++++++++++++++--
include/crm/common/ipc_controld.h | 13 +++++++++
include/crm_internal.h | 1 +
lib/common/ipc_controld.c | 53 ++++++++++++++++++++++++++++++++++++
4 files changed, 109 insertions(+), 2 deletions(-)
diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c
index 0be04d0..423f006 100644
--- a/daemons/controld/controld_messages.c
+++ b/daemons/controld/controld_messages.c
@@ -375,10 +375,11 @@ relay_message(xmlNode * msg, gboolean originated_locally)
is_local = 0;
} else if (is_for_crm) {
- if (safe_str_eq(task, CRM_OP_NODE_INFO)) {
+ if (safe_str_eq(task, CRM_OP_NODE_INFO)
+ || safe_str_eq(task, PCMK__CONTROLD_CMD_NODES)) {
/* Node info requests do not specify a host, which is normally
* treated as "all hosts", because the whole point is that the
- * client doesn't know the local node name. Always handle these
+ * client may not know the local node name. Always handle these
* requests locally.
*/
is_local = 1;
@@ -784,6 +785,42 @@ handle_ping(xmlNode *msg)
}
/*!
+ * \brief Handle a PCMK__CONTROLD_CMD_NODES message
+ *
+ * \return Next FSA input
+ */
+static enum crmd_fsa_input
+handle_node_list(xmlNode *request)
+{
+ GHashTableIter iter;
+ crm_node_t *node = NULL;
+ xmlNode *reply = NULL;
+ xmlNode *reply_data = NULL;
+
+ // Create message data for reply
+ reply_data = create_xml_node(NULL, XML_CIB_TAG_NODES);
+ g_hash_table_iter_init(&iter, crm_peer_cache);
+ while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
+ xmlNode *xml = create_xml_node(reply_data, XML_CIB_TAG_NODE);
+
+ crm_xml_add_ll(xml, XML_ATTR_ID, (long long) node->id); // uint32_t
+ crm_xml_add(xml, XML_ATTR_UNAME, node->uname);
+ crm_xml_add(xml, XML_NODE_IN_CLUSTER, node->state);
+ }
+
+ // Create and send reply
+ reply = create_reply(request, reply_data);
+ free_xml(reply_data);
+ if (reply) {
+ (void) relay_message(reply, TRUE);
+ free_xml(reply);
+ }
+
+ // Nothing further to do
+ return I_NULL;
+}
+
+/*!
* \brief Handle a CRM_OP_NODE_INFO request
*
* \param[in] msg Message XML
@@ -1080,6 +1117,9 @@ handle_request(xmlNode *stored_msg, enum crmd_fsa_cause cause)
remote_ra_process_maintenance_nodes(xml);
+ } else if (strcmp(op, PCMK__CONTROLD_CMD_NODES) == 0) {
+ return handle_node_list(stored_msg);
+
/*========== (NOT_DC)-Only Actions ==========*/
} else if (!AM_I_DC) {
diff --git a/include/crm/common/ipc_controld.h b/include/crm/common/ipc_controld.h
index 0ebabfc..b817357 100644
--- a/include/crm/common/ipc_controld.h
+++ b/include/crm/common/ipc_controld.h
@@ -22,6 +22,7 @@ extern "C" {
*/
#include <stdbool.h> // bool
+#include <glib.h> // GList
#include <libxml/tree.h> // xmlNode
#include <crm/common/ipc.h> // pcmk_ipc_api_t
@@ -32,8 +33,16 @@ enum pcmk_controld_api_reply {
pcmk_controld_reply_info,
pcmk_controld_reply_resource,
pcmk_controld_reply_ping,
+ pcmk_controld_reply_nodes,
};
+// Node information passed with pcmk_controld_reply_nodes
+typedef struct {
+ uint32_t id;
+ const char *uname;
+ const char *state;
+} pcmk_controld_api_node_t;
+
/*!
* Controller reply passed to event callback
*
@@ -72,6 +81,9 @@ typedef struct {
const char *fsa_state;
const char *result;
} ping;
+
+ // pcmk_controld_reply_nodes
+ GList *nodes; // list of pcmk_controld_api_node_t *
} data;
} pcmk_controld_api_reply_t;
@@ -88,6 +100,7 @@ int pcmk_controld_api_refresh(pcmk_ipc_api_t *api, const char *target_node,
const char *provider, const char *type,
bool cib_only);
int pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name);
+int pcmk_controld_api_list_nodes(pcmk_ipc_api_t *api);
int pcmk_controld_api_shutdown(pcmk_ipc_api_t *api, const char *node_name);
int pcmk_controld_api_start_election(pcmk_ipc_api_t *api);
unsigned int pcmk_controld_api_replies_expected(pcmk_ipc_api_t *api);
diff --git a/include/crm_internal.h b/include/crm_internal.h
index fd56fc6..cf8999f 100644
--- a/include/crm_internal.h
+++ b/include/crm_internal.h
@@ -122,6 +122,7 @@ pid_t pcmk_locate_sbd(void);
#define PCMK__ATTRD_CMD_SYNC_RESPONSE "sync-response"
#define PCMK__ATTRD_CMD_CLEAR_FAILURE "clear-failure"
+#define PCMK__CONTROLD_CMD_NODES "list-nodes"
/*
* Environment variables used by Pacemaker
diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c
index 22bb733..a733dd3 100644
--- a/lib/common/ipc_controld.c
+++ b/lib/common/ipc_controld.c
@@ -120,6 +120,28 @@ set_ping_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
data->data.ping.result = crm_element_value(msg_data, XML_PING_ATTR_STATUS);
}
+static void
+set_nodes_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
+{
+ pcmk_controld_api_node_t *node_info;
+
+ data->reply_type = pcmk_controld_reply_nodes;
+ for (xmlNode *node = first_named_child(msg_data, XML_CIB_TAG_NODE);
+ node != NULL; node = crm_next_same_xml(node)) {
+
+ long long id_ll = 0;
+
+ node_info = calloc(1, sizeof(pcmk_controld_api_node_t));
+ crm_element_value_ll(node, XML_ATTR_ID, &id_ll);
+ if (id_ll > 0) {
+ node_info->id = id_ll;
+ }
+ node_info->uname = crm_element_value(node, XML_ATTR_UNAME);
+ node_info->state = crm_element_value(node, XML_NODE_IN_CLUSTER);
+ data->data.nodes = g_list_prepend(data->data.nodes, node_info);
+ }
+}
+
static bool
reply_expected(pcmk_ipc_api_t *api, xmlNode *request)
{
@@ -201,6 +223,9 @@ dispatch(pcmk_ipc_api_t *api, xmlNode *reply)
} else if (!strcmp(value, CRM_OP_PING)) {
set_ping_data(&reply_data, msg_data);
+ } else if (!strcmp(value, PCMK__CONTROLD_CMD_NODES)) {
+ set_nodes_data(&reply_data, msg_data);
+
} else {
crm_debug("Unrecognizable controller message: unknown command '%s'",
value);
@@ -210,6 +235,11 @@ dispatch(pcmk_ipc_api_t *api, xmlNode *reply)
}
pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);
+
+ // Free any reply data that was allocated
+ if (safe_str_eq(value, PCMK__CONTROLD_CMD_NODES)) {
+ g_list_free_full(reply_data.data.nodes, free);
+ }
}
pcmk__ipc_methods_t *
@@ -376,6 +406,29 @@ pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name)
}
/*!
+ * \brief Ask the controller for cluster information
+ *
+ * \param[in] api Controller connection
+ *
+ * \return Standard Pacemaker return code
+ * \note Event callback will get a reply of type pcmk_controld_reply_nodes.
+ */
+int
+pcmk_controld_api_list_nodes(pcmk_ipc_api_t *api)
+{
+ xmlNode *request;
+ int rc = EINVAL;
+
+ request = create_controller_request(api, PCMK__CONTROLD_CMD_NODES, NULL,
+ NULL);
+ if (request != NULL) {
+ rc = send_controller_request(api, request, true);
+ free_xml(request);
+ }
+ return rc;
+}
+
+/*!
* \internal
* \brief Ask the controller to shut down
*
--
1.8.3.1
From 74e2d8d18bf534c1ec6f0e0f44a90772d393a553 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Thu, 2 Jul 2020 11:51:56 -0500
Subject: [PATCH 2/3] Refactor: functionize numeric comparisons of strings
This moves the guts of sort_node_uname() from libpe_status to a new function,
pcmk_numeric_strcasecmp(), in libcrmcommon, so it can be used with strings and
not just pe_node_t objects.
---
include/crm/common/util.h | 1 +
lib/common/strings.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++
lib/pengine/utils.c | 50 ++----------------------------------
3 files changed, 68 insertions(+), 48 deletions(-)
diff --git a/include/crm/common/util.h b/include/crm/common/util.h
index 67d74d2..bb97b0a 100644
--- a/include/crm/common/util.h
+++ b/include/crm/common/util.h
@@ -59,6 +59,7 @@ gboolean crm_strcase_equal(gconstpointer a, gconstpointer b);
char *crm_strdup_printf(char const *format, ...) __attribute__ ((__format__ (__printf__, 1, 2)));
int pcmk__parse_ll_range(const char *srcstring, long long *start, long long *end);
gboolean pcmk__str_in_list(GList *lst, const gchar *s);
+int pcmk_numeric_strcasecmp(const char *s1, const char *s2);
# define safe_str_eq(a, b) crm_str_eq(a, b, FALSE)
# define crm_str_hash g_str_hash_traditional
diff --git a/lib/common/strings.c b/lib/common/strings.c
index 4562738..bd68ccf 100644
--- a/lib/common/strings.c
+++ b/lib/common/strings.c
@@ -16,6 +16,7 @@
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
+#include <ctype.h>
#include <limits.h>
#include <bzlib.h>
#include <sys/types.h>
@@ -715,3 +716,67 @@ pcmk__str_none_of(const char *s, ...)
return g_list_find_custom(lst, s, (GCompareFunc) strcmp) != NULL;
}
+
+/*
+ * \brief Sort strings, with numeric portions sorted numerically
+ *
+ * Sort two strings case-insensitively like strcasecmp(), but with any numeric
+ * portions of the string sorted numerically. This is particularly useful for
+ * node names (for example, "node10" will sort higher than "node9" but lower
+ * than "remotenode9").
+ *
+ * \param[in] s1 First string to compare (must not be NULL)
+ * \param[in] s2 Second string to compare (must not be NULL)
+ *
+ * \retval -1 \p s1 comes before \p s2
+ * \retval 0 \p s1 and \p s2 are equal
+ * \retval 1 \p s1 comes after \p s2
+ */
+int
+pcmk_numeric_strcasecmp(const char *s1, const char *s2)
+{
+ while (*s1 && *s2) {
+ if (isdigit(*s1) && isdigit(*s2)) {
+ // If node names contain a number, sort numerically
+
+ char *end1 = NULL;
+ char *end2 = NULL;
+ long num1 = strtol(s1, &end1, 10);
+ long num2 = strtol(s2, &end2, 10);
+
+ // allow ordering e.g. 007 > 7
+ size_t len1 = end1 - s1;
+ size_t len2 = end2 - s2;
+
+ if (num1 < num2) {
+ return -1;
+ } else if (num1 > num2) {
+ return 1;
+ } else if (len1 < len2) {
+ return -1;
+ } else if (len1 > len2) {
+ return 1;
+ }
+ s1 = end1;
+ s2 = end2;
+ } else {
+ // Compare non-digits case-insensitively
+ int lower1 = tolower(*s1);
+ int lower2 = tolower(*s2);
+
+ if (lower1 < lower2) {
+ return -1;
+ } else if (lower1 > lower2) {
+ return 1;
+ }
+ ++s1;
+ ++s2;
+ }
+ }
+ if (!*s1 && *s2) {
+ return -1;
+ } else if (*s1 && !*s2) {
+ return 1;
+ }
+ return 0;
+}
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index ce3c260..584def4 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -13,7 +13,6 @@
#include <crm/common/xml.h>
#include <crm/common/util.h>
-#include <ctype.h>
#include <glib.h>
#include <stdbool.h>
@@ -214,53 +213,8 @@ pe__node_list2table(GList *list)
gint
sort_node_uname(gconstpointer a, gconstpointer b)
{
- const char *name_a = ((const pe_node_t *) a)->details->uname;
- const char *name_b = ((const pe_node_t *) b)->details->uname;
-
- while (*name_a && *name_b) {
- if (isdigit(*name_a) && isdigit(*name_b)) {
- // If node names contain a number, sort numerically
-
- char *end_a = NULL;
- char *end_b = NULL;
- long num_a = strtol(name_a, &end_a, 10);
- long num_b = strtol(name_b, &end_b, 10);
-
- // allow ordering e.g. 007 > 7
- size_t len_a = end_a - name_a;
- size_t len_b = end_b - name_b;
-
- if (num_a < num_b) {
- return -1;
- } else if (num_a > num_b) {
- return 1;
- } else if (len_a < len_b) {
- return -1;
- } else if (len_a > len_b) {
- return 1;
- }
- name_a = end_a;
- name_b = end_b;
- } else {
- // Compare non-digits case-insensitively
- int lower_a = tolower(*name_a);
- int lower_b = tolower(*name_b);
-
- if (lower_a < lower_b) {
- return -1;
- } else if (lower_a > lower_b) {
- return 1;
- }
- ++name_a;
- ++name_b;
- }
- }
- if (!*name_a && *name_b) {
- return -1;
- } else if (*name_a && !*name_b) {
- return 1;
- }
- return 0;
+ return pcmk_numeric_strcasecmp(((const pe_node_t *) a)->details->uname,
+ ((const pe_node_t *) b)->details->uname);
}
/*!
--
1.8.3.1
From 8461509158e06365122dc741c527c83c94e966ce Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Fri, 24 Apr 2020 19:35:19 -0500
Subject: [PATCH 3/3] Fix: tools: crm_node -l and -p now work from Pacemaker
Remote nodes
crm_node now asks the controller for the cluster node list, instead of
pacemakerd. This allows it to work from Pacemaker Remote nodes, since
controller IPC is proxied but pacemakerd IPC is not.
---
tools/crm_node.c | 176 +++++++++++++++++++++----------------------------------
1 file changed, 67 insertions(+), 109 deletions(-)
diff --git a/tools/crm_node.c b/tools/crm_node.c
index 57c2ee5..146454d 100644
--- a/tools/crm_node.c
+++ b/tools/crm_node.c
@@ -130,6 +130,16 @@ remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError *
return TRUE;
}
+static gint
+sort_node(gconstpointer a, gconstpointer b)
+{
+ const pcmk_controld_api_node_t *node_a = a;
+ const pcmk_controld_api_node_t *node_b = b;
+
+ return pcmk_numeric_strcasecmp((node_a->uname? node_a->uname : ""),
+ (node_b->uname? node_b->uname : ""));
+}
+
static void
controller_event_cb(pcmk_ipc_api_t *controld_api,
enum pcmk_ipc_event event_type, crm_exit_t status,
@@ -157,15 +167,16 @@ controller_event_cb(pcmk_ipc_api_t *controld_api,
crm_exit_str(status));
goto done;
}
- if (reply->reply_type != pcmk_controld_reply_info) {
- fprintf(stderr, "error: Unknown reply type %d from controller\n",
- reply->reply_type);
- goto done;
- }
// Parse desired info from reply and display to user
switch (options.command) {
case 'i':
+ if (reply->reply_type != pcmk_controld_reply_info) {
+ fprintf(stderr,
+ "error: Unknown reply type %d from controller\n",
+ reply->reply_type);
+ goto done;
+ }
if (reply->data.node_info.id == 0) {
fprintf(stderr,
"error: Controller reply did not contain node ID\n");
@@ -177,6 +188,12 @@ controller_event_cb(pcmk_ipc_api_t *controld_api,
case 'n':
case 'N':
+ if (reply->reply_type != pcmk_controld_reply_info) {
+ fprintf(stderr,
+ "error: Unknown reply type %d from controller\n",
+ reply->reply_type);
+ goto done;
+ }
if (reply->data.node_info.uname == NULL) {
fprintf(stderr, "Node is not known to cluster\n");
exit_code = CRM_EX_NOHOST;
@@ -186,6 +203,12 @@ controller_event_cb(pcmk_ipc_api_t *controld_api,
break;
case 'q':
+ if (reply->reply_type != pcmk_controld_reply_info) {
+ fprintf(stderr,
+ "error: Unknown reply type %d from controller\n",
+ reply->reply_type);
+ goto done;
+ }
printf("%d\n", reply->data.node_info.have_quorum);
if (!(reply->data.node_info.have_quorum)) {
exit_code = CRM_EX_QUORUM;
@@ -193,6 +216,36 @@ controller_event_cb(pcmk_ipc_api_t *controld_api,
}
break;
+ case 'l':
+ case 'p':
+ if (reply->reply_type != pcmk_controld_reply_nodes) {
+ fprintf(stderr,
+ "error: Unknown reply type %d from controller\n",
+ reply->reply_type);
+ goto done;
+ }
+ reply->data.nodes = g_list_sort(reply->data.nodes, sort_node);
+ for (GList *node_iter = reply->data.nodes;
+ node_iter != NULL; node_iter = node_iter->next) {
+
+ pcmk_controld_api_node_t *node = node_iter->data;
+ const char *uname = (node->uname? node->uname : "");
+ const char *state = (node->state? node->state : "");
+
+ if (options.command == 'l') {
+ printf("%lu %s %s\n",
+ (unsigned long) node->id, uname, state);
+
+ // i.e. CRM_NODE_MEMBER, but we don't want to include cluster.h
+ } else if (!strcmp(state, "member")) {
+ printf("%s ", uname);
+ }
+ }
+ if (options.command == 'p') {
+ printf("\n");
+ }
+ break;
+
default:
fprintf(stderr, "internal error: Controller reply not expected\n");
exit_code = CRM_EX_SOFTWARE;
@@ -207,7 +260,7 @@ done:
}
static void
-run_controller_mainloop(uint32_t nodeid)
+run_controller_mainloop(uint32_t nodeid, bool list_nodes)
{
pcmk_ipc_api_t *controld_api = NULL;
int rc;
@@ -233,7 +286,11 @@ run_controller_mainloop(uint32_t nodeid)
return;
}
- rc = pcmk_controld_api_node_info(controld_api, nodeid);
+ if (list_nodes) {
+ rc = pcmk_controld_api_list_nodes(controld_api);
+ } else {
+ rc = pcmk_controld_api_node_info(controld_api, nodeid);
+ }
if (rc != pcmk_rc_ok) {
fprintf(stderr, "error: Could not ping controller: %s\n",
pcmk_rc_str(rc));
@@ -263,7 +320,7 @@ print_node_name(void)
} else {
// Otherwise ask the controller
- run_controller_mainloop(0);
+ run_controller_mainloop(0, false);
}
}
@@ -444,105 +501,6 @@ remove_node(const char *target_uname)
exit_code = CRM_EX_OK;
}
-static gint
-compare_node_xml(gconstpointer a, gconstpointer b)
-{
- const char *a_name = crm_element_value((xmlNode*) a, "uname");
- const char *b_name = crm_element_value((xmlNode*) b, "uname");
-
- return strcmp((a_name? a_name : ""), (b_name? b_name : ""));
-}
-
-static int
-node_mcp_dispatch(const char *buffer, ssize_t length, gpointer userdata)
-{
- GList *nodes = NULL;
- xmlNode *node = NULL;
- xmlNode *msg = string2xml(buffer);
- const char *uname;
- const char *state;
-
- if (msg == NULL) {
- fprintf(stderr, "error: Could not understand pacemakerd response\n");
- exit_code = CRM_EX_PROTOCOL;
- g_main_loop_quit(mainloop);
- return 0;
- }
-
- crm_log_xml_trace(msg, "message");
-
- for (node = __xml_first_child(msg); node != NULL; node = __xml_next(node)) {
- nodes = g_list_insert_sorted(nodes, node, compare_node_xml);
- }
-
- for (GList *iter = nodes; iter; iter = iter->next) {
- node = (xmlNode*) iter->data;
- uname = crm_element_value(node, "uname");
- state = crm_element_value(node, "state");
-
- if (options.command == 'l') {
- int id = 0;
-
- crm_element_value_int(node, "id", &id);
- printf("%d %s %s\n", id, (uname? uname : ""), (state? state : ""));
-
- // This is CRM_NODE_MEMBER but we don't want to include cluster header
- } else if ((options.command == 'p') && safe_str_eq(state, "member")) {
- printf("%s ", (uname? uname : ""));
- }
- }
- if (options.command == 'p') {
- fprintf(stdout, "\n");
- }
-
- free_xml(msg);
- exit_code = CRM_EX_OK;
- g_main_loop_quit(mainloop);
- return 0;
-}
-
-static void
-lost_pacemakerd(gpointer user_data)
-{
- fprintf(stderr, "error: Lost connection to cluster\n");
- exit_code = CRM_EX_DISCONNECT;
- g_main_loop_quit(mainloop);
-}
-
-static void
-run_pacemakerd_mainloop(void)
-{
- crm_ipc_t *ipc = NULL;
- xmlNode *poke = NULL;
- mainloop_io_t *source = NULL;
-
- struct ipc_client_callbacks ipc_callbacks = {
- .dispatch = node_mcp_dispatch,
- .destroy = lost_pacemakerd
- };
-
- source = mainloop_add_ipc_client(CRM_SYSTEM_MCP, G_PRIORITY_DEFAULT, 0,
- NULL, &ipc_callbacks);
- ipc = mainloop_get_ipc_client(source);
- if (ipc == NULL) {
- fprintf(stderr,
- "error: Could not connect to cluster (is it running?)\n");
- exit_code = CRM_EX_DISCONNECT;
- return;
- }
-
- // Sending anything will get us a list of nodes
- poke = create_xml_node(NULL, "poke");
- crm_ipc_send(ipc, poke, 0, 0, NULL);
- free_xml(poke);
-
- // Handle reply via node_mcp_dispatch()
- mainloop = g_main_loop_new(NULL, FALSE);
- g_main_loop_run(mainloop);
- g_main_loop_unref(mainloop);
- mainloop = NULL;
-}
-
static GOptionContext *
build_arg_context(pcmk__common_args_t *args, GOptionGroup *group) {
GOptionContext *context = NULL;
@@ -627,11 +585,11 @@ main(int argc, char **argv)
case 'i':
case 'q':
case 'N':
- run_controller_mainloop(options.nodeid);
+ run_controller_mainloop(options.nodeid, false);
break;
case 'l':
case 'p':
- run_pacemakerd_mainloop();
+ run_controller_mainloop(0, true);
break;
default:
break;
--
1.8.3.1