diff --git a/.gitignore b/.gitignore index 6e6662f..07d8dae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ SOURCES/nagios-agents-metadata-105ab8a.tar.gz -SOURCES/pacemaker-2deceaa.tar.gz +SOURCES/pacemaker-31aa4f5.tar.gz diff --git a/.pacemaker.metadata b/.pacemaker.metadata index 6a9af04..a932749 100644 --- a/.pacemaker.metadata +++ b/.pacemaker.metadata @@ -1,2 +1,2 @@ ea6c0a27fd0ae8ce02f84a11f08a0d79377041c3 SOURCES/nagios-agents-metadata-105ab8a.tar.gz -78c94fdcf59cfb064d4433e1b8f71fd856eeec5f SOURCES/pacemaker-2deceaa.tar.gz +a1e8aa58fa9667d852676ff883c666729b51761f SOURCES/pacemaker-31aa4f5.tar.gz diff --git a/SOURCES/001-rules.patch b/SOURCES/001-rules.patch deleted file mode 100644 index 0133975..0000000 --- a/SOURCES/001-rules.patch +++ /dev/null @@ -1,4947 +0,0 @@ -From 2f10dde2f2a0ac7a3d74cb2f398be1deaba75615 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Mon, 6 Apr 2020 11:22:50 -0400 -Subject: [PATCH 01/17] Feature: scheduler: Add new expression_type values. - ---- - include/crm/pengine/rules.h | 4 +++- - lib/pengine/rules.c | 6 ++++++ - 2 files changed, 9 insertions(+), 1 deletion(-) - -diff --git a/include/crm/pengine/rules.h b/include/crm/pengine/rules.h -index ebd3148..37f092b 100644 ---- a/include/crm/pengine/rules.h -+++ b/include/crm/pengine/rules.h -@@ -28,7 +28,9 @@ enum expression_type { - loc_expr, - role_expr, - time_expr, -- version_expr -+ version_expr, -+ rsc_expr, -+ op_expr - }; - - typedef struct pe_re_match_data { -diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c -index fa9a222..130bada 100644 ---- a/lib/pengine/rules.c -+++ b/lib/pengine/rules.c -@@ -189,6 +189,12 @@ find_expression_type(xmlNode * expr) - if (safe_str_eq(tag, "date_expression")) { - return time_expr; - -+ } else if (safe_str_eq(tag, "rsc_expression")) { -+ return rsc_expr; -+ -+ } else if (safe_str_eq(tag, "op_expression")) { -+ return op_expr; -+ - } else if (safe_str_eq(tag, XML_TAG_RULE)) { - return nested_rule; - --- -1.8.3.1 - - -From bc7491e5226af2a2e7f1a9b2d61892d3af0767fe Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Fri, 3 Apr 2020 15:03:23 -0400 -Subject: [PATCH 02/17] Refactor: scheduler: Add new pe__eval_*_expr functions. - -These new functions all take the same input arguments - an xmlNodePtr -and a pe_rule_eval_data_t. This latter type holds all the parameters -that could possibly be useful for evaluating some rule. Most functions -will only need a few items out of this structure. - -Then, implement pe_test_*_expression in terms of these new functions. ---- - include/crm/pengine/common.h | 37 ++- - include/crm/pengine/rules.h | 13 - - include/crm/pengine/rules_internal.h | 5 + - lib/pengine/rules.c | 592 +++++++++++++++++++---------------- - 4 files changed, 363 insertions(+), 284 deletions(-) - -diff --git a/include/crm/pengine/common.h b/include/crm/pengine/common.h -index 48c2b66..3a770b7 100644 ---- a/include/crm/pengine/common.h -+++ b/include/crm/pengine/common.h -@@ -1,5 +1,5 @@ - /* -- * Copyright 2004-2019 the Pacemaker project contributors -+ * Copyright 2004-2020 the Pacemaker project contributors - * - * The version control history for this file may have further details. - * -@@ -15,6 +15,9 @@ extern "C" { - #endif - - # include -+# include -+ -+# include - - extern gboolean was_processing_error; - extern gboolean was_processing_warning; -@@ -131,6 +134,38 @@ recovery2text(enum rsc_recovery_type type) - return "Unknown"; - } - -+typedef struct pe_re_match_data { -+ char *string; -+ int nregs; -+ regmatch_t *pmatch; -+} pe_re_match_data_t; -+ -+typedef struct pe_match_data { -+ pe_re_match_data_t *re; -+ GHashTable *params; -+ GHashTable *meta; -+} pe_match_data_t; -+ -+typedef struct pe_rsc_eval_data { -+ const char *standard; -+ const char *provider; -+ const char *agent; -+} pe_rsc_eval_data_t; -+ -+typedef struct pe_op_eval_data { -+ const char *op_name; -+ guint interval; -+} pe_op_eval_data_t; -+ -+typedef struct pe_rule_eval_data { -+ GHashTable *node_hash; -+ enum rsc_role_e role; -+ crm_time_t *now; -+ pe_match_data_t *match_data; -+ pe_rsc_eval_data_t *rsc_data; -+ pe_op_eval_data_t *op_data; -+} pe_rule_eval_data_t; -+ - #ifdef __cplusplus - } - #endif -diff --git a/include/crm/pengine/rules.h b/include/crm/pengine/rules.h -index 37f092b..d7bdbf9 100644 ---- a/include/crm/pengine/rules.h -+++ b/include/crm/pengine/rules.h -@@ -15,7 +15,6 @@ extern "C" { - #endif - - # include --# include - - # include - # include -@@ -33,18 +32,6 @@ enum expression_type { - op_expr - }; - --typedef struct pe_re_match_data { -- char *string; -- int nregs; -- regmatch_t *pmatch; --} pe_re_match_data_t; -- --typedef struct pe_match_data { -- pe_re_match_data_t *re; -- GHashTable *params; -- GHashTable *meta; --} pe_match_data_t; -- - enum expression_type find_expression_type(xmlNode * expr); - - gboolean pe_evaluate_rules(xmlNode *ruleset, GHashTable *node_hash, -diff --git a/include/crm/pengine/rules_internal.h b/include/crm/pengine/rules_internal.h -index fd65c1e..8a22108 100644 ---- a/include/crm/pengine/rules_internal.h -+++ b/include/crm/pengine/rules_internal.h -@@ -21,6 +21,11 @@ void pe_free_alert_list(GListPtr alert_list); - - crm_time_t *pe_parse_xml_duration(crm_time_t * start, xmlNode * duration_spec); - -+gboolean pe__eval_attr_expr(xmlNode *expr, pe_rule_eval_data_t *rule_data); -+int pe__eval_date_expr(xmlNode *expr, pe_rule_eval_data_t *rule_data, -+ crm_time_t *next_change); -+gboolean pe__eval_role_expr(xmlNode *expr, pe_rule_eval_data_t *rule_data); -+ - int pe_eval_date_expression(xmlNode *time_expr, - crm_time_t *now, - crm_time_t *next_change); -diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c -index 130bada..3f316c2 100644 ---- a/lib/pengine/rules.c -+++ b/lib/pengine/rules.c -@@ -219,201 +219,34 @@ find_expression_type(xmlNode * expr) - } - - gboolean --pe_test_role_expression(xmlNode * expr, enum rsc_role_e role, crm_time_t * now) -+pe_test_role_expression(xmlNode *expr, enum rsc_role_e role, crm_time_t *now) - { -- gboolean accept = FALSE; -- const char *op = NULL; -- const char *value = NULL; -- -- if (role == RSC_ROLE_UNKNOWN) { -- return accept; -- } -- -- value = crm_element_value(expr, XML_EXPR_ATTR_VALUE); -- op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION); -- -- if (safe_str_eq(op, "defined")) { -- if (role > RSC_ROLE_STARTED) { -- accept = TRUE; -- } -- -- } else if (safe_str_eq(op, "not_defined")) { -- if (role < RSC_ROLE_SLAVE && role > RSC_ROLE_UNKNOWN) { -- accept = TRUE; -- } -- -- } else if (safe_str_eq(op, "eq")) { -- if (text2role(value) == role) { -- accept = TRUE; -- } -- -- } else if (safe_str_eq(op, "ne")) { -- // Test "ne" only with promotable clone roles -- if (role < RSC_ROLE_SLAVE && role > RSC_ROLE_UNKNOWN) { -- accept = FALSE; -- -- } else if (text2role(value) != role) { -- accept = TRUE; -- } -- } -- return accept; -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = NULL, -+ .role = role, -+ .now = now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ -+ return pe__eval_role_expr(expr, &rule_data); - } - - gboolean - pe_test_attr_expression(xmlNode *expr, GHashTable *hash, crm_time_t *now, - pe_match_data_t *match_data) - { -- gboolean accept = FALSE; -- gboolean attr_allocated = FALSE; -- int cmp = 0; -- const char *h_val = NULL; -- GHashTable *table = NULL; -- -- const char *op = NULL; -- const char *type = NULL; -- const char *attr = NULL; -- const char *value = NULL; -- const char *value_source = NULL; -- -- attr = crm_element_value(expr, XML_EXPR_ATTR_ATTRIBUTE); -- op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION); -- value = crm_element_value(expr, XML_EXPR_ATTR_VALUE); -- type = crm_element_value(expr, XML_EXPR_ATTR_TYPE); -- value_source = crm_element_value(expr, XML_EXPR_ATTR_VALUE_SOURCE); -- -- if (attr == NULL || op == NULL) { -- pe_err("Invalid attribute or operation in expression" -- " (\'%s\' \'%s\' \'%s\')", crm_str(attr), crm_str(op), crm_str(value)); -- return FALSE; -- } -- -- if (match_data) { -- if (match_data->re) { -- char *resolved_attr = pe_expand_re_matches(attr, match_data->re); -- -- if (resolved_attr) { -- attr = (const char *) resolved_attr; -- attr_allocated = TRUE; -- } -- } -- -- if (safe_str_eq(value_source, "param")) { -- table = match_data->params; -- } else if (safe_str_eq(value_source, "meta")) { -- table = match_data->meta; -- } -- } -- -- if (table) { -- const char *param_name = value; -- const char *param_value = NULL; -- -- if (param_name && param_name[0]) { -- if ((param_value = (const char *)g_hash_table_lookup(table, param_name))) { -- value = param_value; -- } -- } -- } -- -- if (hash != NULL) { -- h_val = (const char *)g_hash_table_lookup(hash, attr); -- } -- -- if (attr_allocated) { -- free((char *)attr); -- attr = NULL; -- } -- -- if (value != NULL && h_val != NULL) { -- if (type == NULL) { -- if (safe_str_eq(op, "lt") -- || safe_str_eq(op, "lte") -- || safe_str_eq(op, "gt") -- || safe_str_eq(op, "gte")) { -- type = "number"; -- -- } else { -- type = "string"; -- } -- crm_trace("Defaulting to %s based comparison for '%s' op", type, op); -- } -- -- if (safe_str_eq(type, "string")) { -- cmp = strcasecmp(h_val, value); -- -- } else if (safe_str_eq(type, "number")) { -- int h_val_f = crm_parse_int(h_val, NULL); -- int value_f = crm_parse_int(value, NULL); -- -- if (h_val_f < value_f) { -- cmp = -1; -- } else if (h_val_f > value_f) { -- cmp = 1; -- } else { -- cmp = 0; -- } -- -- } else if (safe_str_eq(type, "version")) { -- cmp = compare_version(h_val, value); -- -- } -- -- } else if (value == NULL && h_val == NULL) { -- cmp = 0; -- } else if (value == NULL) { -- cmp = 1; -- } else { -- cmp = -1; -- } -- -- if (safe_str_eq(op, "defined")) { -- if (h_val != NULL) { -- accept = TRUE; -- } -- -- } else if (safe_str_eq(op, "not_defined")) { -- if (h_val == NULL) { -- accept = TRUE; -- } -- -- } else if (safe_str_eq(op, "eq")) { -- if ((h_val == value) || cmp == 0) { -- accept = TRUE; -- } -- -- } else if (safe_str_eq(op, "ne")) { -- if ((h_val == NULL && value != NULL) -- || (h_val != NULL && value == NULL) -- || cmp != 0) { -- accept = TRUE; -- } -- -- } else if (value == NULL || h_val == NULL) { -- // The comparison is meaningless from this point on -- accept = FALSE; -- -- } else if (safe_str_eq(op, "lt")) { -- if (cmp < 0) { -- accept = TRUE; -- } -- -- } else if (safe_str_eq(op, "lte")) { -- if (cmp <= 0) { -- accept = TRUE; -- } -- -- } else if (safe_str_eq(op, "gt")) { -- if (cmp > 0) { -- accept = TRUE; -- } -- -- } else if (safe_str_eq(op, "gte")) { -- if (cmp >= 0) { -- accept = TRUE; -- } -- } -- -- return accept; -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = hash, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = now, -+ .match_data = match_data, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ -+ return pe__eval_attr_expr(expr, &rule_data); - } - - /* As per the nethack rules: -@@ -587,10 +420,18 @@ pe_parse_xml_duration(crm_time_t * start, xmlNode * duration_spec) - * \return TRUE if date expression is in effect at given time, FALSE otherwise - */ - gboolean --pe_test_date_expression(xmlNode *time_expr, crm_time_t *now, -- crm_time_t *next_change) -+pe_test_date_expression(xmlNode *expr, crm_time_t *now, crm_time_t *next_change) - { -- switch (pe_eval_date_expression(time_expr, now, next_change)) { -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = NULL, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ -+ switch (pe__eval_date_expr(expr, &rule_data, next_change)) { - case pcmk_rc_within_range: - case pcmk_rc_ok: - return TRUE; -@@ -623,86 +464,18 @@ crm_time_set_if_earlier(crm_time_t *next_change, crm_time_t *t) - * \return Standard Pacemaker return code - */ - int --pe_eval_date_expression(xmlNode *time_expr, crm_time_t *now, -- crm_time_t *next_change) -+pe_eval_date_expression(xmlNode *expr, crm_time_t *now, crm_time_t *next_change) - { -- crm_time_t *start = NULL; -- crm_time_t *end = NULL; -- const char *value = NULL; -- const char *op = crm_element_value(time_expr, "operation"); -- -- xmlNode *duration_spec = NULL; -- xmlNode *date_spec = NULL; -- -- // "undetermined" will also be returned for parsing errors -- int rc = pcmk_rc_undetermined; -- -- crm_trace("Testing expression: %s", ID(time_expr)); -- -- duration_spec = first_named_child(time_expr, "duration"); -- date_spec = first_named_child(time_expr, "date_spec"); -- -- value = crm_element_value(time_expr, "start"); -- if (value != NULL) { -- start = crm_time_new(value); -- } -- value = crm_element_value(time_expr, "end"); -- if (value != NULL) { -- end = crm_time_new(value); -- } -- -- if (start != NULL && end == NULL && duration_spec != NULL) { -- end = pe_parse_xml_duration(start, duration_spec); -- } -- -- if ((op == NULL) || safe_str_eq(op, "in_range")) { -- if ((start == NULL) && (end == NULL)) { -- // in_range requires at least one of start or end -- } else if ((start != NULL) && (crm_time_compare(now, start) < 0)) { -- rc = pcmk_rc_before_range; -- crm_time_set_if_earlier(next_change, start); -- } else if ((end != NULL) && (crm_time_compare(now, end) > 0)) { -- rc = pcmk_rc_after_range; -- } else { -- rc = pcmk_rc_within_range; -- if (end && next_change) { -- // Evaluation doesn't change until second after end -- crm_time_add_seconds(end, 1); -- crm_time_set_if_earlier(next_change, end); -- } -- } -- -- } else if (safe_str_eq(op, "date_spec")) { -- rc = pe_cron_range_satisfied(now, date_spec); -- // @TODO set next_change appropriately -- -- } else if (safe_str_eq(op, "gt")) { -- if (start == NULL) { -- // gt requires start -- } else if (crm_time_compare(now, start) > 0) { -- rc = pcmk_rc_within_range; -- } else { -- rc = pcmk_rc_before_range; -- -- // Evaluation doesn't change until second after start -- crm_time_add_seconds(start, 1); -- crm_time_set_if_earlier(next_change, start); -- } -- -- } else if (safe_str_eq(op, "lt")) { -- if (end == NULL) { -- // lt requires end -- } else if (crm_time_compare(now, end) < 0) { -- rc = pcmk_rc_within_range; -- crm_time_set_if_earlier(next_change, end); -- } else { -- rc = pcmk_rc_after_range; -- } -- } -- -- crm_time_free(start); -- crm_time_free(end); -- return rc; -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = NULL, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ -+ return pe__eval_date_expr(expr, &rule_data, next_change); - } - - // Information about a block of nvpair elements -@@ -1111,6 +884,285 @@ pe_unpack_versioned_parameters(xmlNode *versioned_params, const char *ra_version - } - #endif - -+gboolean -+pe__eval_attr_expr(xmlNodePtr expr, pe_rule_eval_data_t *rule_data) -+{ -+ gboolean accept = FALSE; -+ gboolean attr_allocated = FALSE; -+ int cmp = 0; -+ const char *h_val = NULL; -+ GHashTable *table = NULL; -+ -+ const char *op = NULL; -+ const char *type = NULL; -+ const char *attr = NULL; -+ const char *value = NULL; -+ const char *value_source = NULL; -+ -+ attr = crm_element_value(expr, XML_EXPR_ATTR_ATTRIBUTE); -+ op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION); -+ value = crm_element_value(expr, XML_EXPR_ATTR_VALUE); -+ type = crm_element_value(expr, XML_EXPR_ATTR_TYPE); -+ value_source = crm_element_value(expr, XML_EXPR_ATTR_VALUE_SOURCE); -+ -+ if (attr == NULL || op == NULL) { -+ pe_err("Invalid attribute or operation in expression" -+ " (\'%s\' \'%s\' \'%s\')", crm_str(attr), crm_str(op), crm_str(value)); -+ return FALSE; -+ } -+ -+ if (rule_data->match_data) { -+ if (rule_data->match_data->re) { -+ char *resolved_attr = pe_expand_re_matches(attr, rule_data->match_data->re); -+ -+ if (resolved_attr) { -+ attr = (const char *) resolved_attr; -+ attr_allocated = TRUE; -+ } -+ } -+ -+ if (safe_str_eq(value_source, "param")) { -+ table = rule_data->match_data->params; -+ } else if (safe_str_eq(value_source, "meta")) { -+ table = rule_data->match_data->meta; -+ } -+ } -+ -+ if (table) { -+ const char *param_name = value; -+ const char *param_value = NULL; -+ -+ if (param_name && param_name[0]) { -+ if ((param_value = (const char *)g_hash_table_lookup(table, param_name))) { -+ value = param_value; -+ } -+ } -+ } -+ -+ if (rule_data->node_hash != NULL) { -+ h_val = (const char *)g_hash_table_lookup(rule_data->node_hash, attr); -+ } -+ -+ if (attr_allocated) { -+ free((char *)attr); -+ attr = NULL; -+ } -+ -+ if (value != NULL && h_val != NULL) { -+ if (type == NULL) { -+ if (safe_str_eq(op, "lt") -+ || safe_str_eq(op, "lte") -+ || safe_str_eq(op, "gt") -+ || safe_str_eq(op, "gte")) { -+ type = "number"; -+ -+ } else { -+ type = "string"; -+ } -+ crm_trace("Defaulting to %s based comparison for '%s' op", type, op); -+ } -+ -+ if (safe_str_eq(type, "string")) { -+ cmp = strcasecmp(h_val, value); -+ -+ } else if (safe_str_eq(type, "number")) { -+ int h_val_f = crm_parse_int(h_val, NULL); -+ int value_f = crm_parse_int(value, NULL); -+ -+ if (h_val_f < value_f) { -+ cmp = -1; -+ } else if (h_val_f > value_f) { -+ cmp = 1; -+ } else { -+ cmp = 0; -+ } -+ -+ } else if (safe_str_eq(type, "version")) { -+ cmp = compare_version(h_val, value); -+ -+ } -+ -+ } else if (value == NULL && h_val == NULL) { -+ cmp = 0; -+ } else if (value == NULL) { -+ cmp = 1; -+ } else { -+ cmp = -1; -+ } -+ -+ if (safe_str_eq(op, "defined")) { -+ if (h_val != NULL) { -+ accept = TRUE; -+ } -+ -+ } else if (safe_str_eq(op, "not_defined")) { -+ if (h_val == NULL) { -+ accept = TRUE; -+ } -+ -+ } else if (safe_str_eq(op, "eq")) { -+ if ((h_val == value) || cmp == 0) { -+ accept = TRUE; -+ } -+ -+ } else if (safe_str_eq(op, "ne")) { -+ if ((h_val == NULL && value != NULL) -+ || (h_val != NULL && value == NULL) -+ || cmp != 0) { -+ accept = TRUE; -+ } -+ -+ } else if (value == NULL || h_val == NULL) { -+ // The comparison is meaningless from this point on -+ accept = FALSE; -+ -+ } else if (safe_str_eq(op, "lt")) { -+ if (cmp < 0) { -+ accept = TRUE; -+ } -+ -+ } else if (safe_str_eq(op, "lte")) { -+ if (cmp <= 0) { -+ accept = TRUE; -+ } -+ -+ } else if (safe_str_eq(op, "gt")) { -+ if (cmp > 0) { -+ accept = TRUE; -+ } -+ -+ } else if (safe_str_eq(op, "gte")) { -+ if (cmp >= 0) { -+ accept = TRUE; -+ } -+ } -+ -+ return accept; -+} -+ -+int -+pe__eval_date_expr(xmlNodePtr expr, pe_rule_eval_data_t *rule_data, crm_time_t *next_change) -+{ -+ crm_time_t *start = NULL; -+ crm_time_t *end = NULL; -+ const char *value = NULL; -+ const char *op = crm_element_value(expr, "operation"); -+ -+ xmlNode *duration_spec = NULL; -+ xmlNode *date_spec = NULL; -+ -+ // "undetermined" will also be returned for parsing errors -+ int rc = pcmk_rc_undetermined; -+ -+ crm_trace("Testing expression: %s", ID(expr)); -+ -+ duration_spec = first_named_child(expr, "duration"); -+ date_spec = first_named_child(expr, "date_spec"); -+ -+ value = crm_element_value(expr, "start"); -+ if (value != NULL) { -+ start = crm_time_new(value); -+ } -+ value = crm_element_value(expr, "end"); -+ if (value != NULL) { -+ end = crm_time_new(value); -+ } -+ -+ if (start != NULL && end == NULL && duration_spec != NULL) { -+ end = pe_parse_xml_duration(start, duration_spec); -+ } -+ -+ if ((op == NULL) || safe_str_eq(op, "in_range")) { -+ if ((start == NULL) && (end == NULL)) { -+ // in_range requires at least one of start or end -+ } else if ((start != NULL) && (crm_time_compare(rule_data->now, start) < 0)) { -+ rc = pcmk_rc_before_range; -+ crm_time_set_if_earlier(next_change, start); -+ } else if ((end != NULL) && (crm_time_compare(rule_data->now, end) > 0)) { -+ rc = pcmk_rc_after_range; -+ } else { -+ rc = pcmk_rc_within_range; -+ if (end && next_change) { -+ // Evaluation doesn't change until second after end -+ crm_time_add_seconds(end, 1); -+ crm_time_set_if_earlier(next_change, end); -+ } -+ } -+ -+ } else if (safe_str_eq(op, "date_spec")) { -+ rc = pe_cron_range_satisfied(rule_data->now, date_spec); -+ // @TODO set next_change appropriately -+ -+ } else if (safe_str_eq(op, "gt")) { -+ if (start == NULL) { -+ // gt requires start -+ } else if (crm_time_compare(rule_data->now, start) > 0) { -+ rc = pcmk_rc_within_range; -+ } else { -+ rc = pcmk_rc_before_range; -+ -+ // Evaluation doesn't change until second after start -+ crm_time_add_seconds(start, 1); -+ crm_time_set_if_earlier(next_change, start); -+ } -+ -+ } else if (safe_str_eq(op, "lt")) { -+ if (end == NULL) { -+ // lt requires end -+ } else if (crm_time_compare(rule_data->now, end) < 0) { -+ rc = pcmk_rc_within_range; -+ crm_time_set_if_earlier(next_change, end); -+ } else { -+ rc = pcmk_rc_after_range; -+ } -+ } -+ -+ crm_time_free(start); -+ crm_time_free(end); -+ return rc; -+} -+ -+gboolean -+pe__eval_role_expr(xmlNodePtr expr, pe_rule_eval_data_t *rule_data) -+{ -+ gboolean accept = FALSE; -+ const char *op = NULL; -+ const char *value = NULL; -+ -+ if (rule_data->role == RSC_ROLE_UNKNOWN) { -+ return accept; -+ } -+ -+ value = crm_element_value(expr, XML_EXPR_ATTR_VALUE); -+ op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION); -+ -+ if (safe_str_eq(op, "defined")) { -+ if (rule_data->role > RSC_ROLE_STARTED) { -+ accept = TRUE; -+ } -+ -+ } else if (safe_str_eq(op, "not_defined")) { -+ if (rule_data->role < RSC_ROLE_SLAVE && rule_data->role > RSC_ROLE_UNKNOWN) { -+ accept = TRUE; -+ } -+ -+ } else if (safe_str_eq(op, "eq")) { -+ if (text2role(value) == rule_data->role) { -+ accept = TRUE; -+ } -+ -+ } else if (safe_str_eq(op, "ne")) { -+ // Test "ne" only with promotable clone roles -+ if (rule_data->role < RSC_ROLE_SLAVE && rule_data->role > RSC_ROLE_UNKNOWN) { -+ accept = FALSE; -+ -+ } else if (text2role(value) != rule_data->role) { -+ accept = TRUE; -+ } -+ } -+ return accept; -+} -+ - // Deprecated functions kept only for backward API compatibility - gboolean test_ruleset(xmlNode *ruleset, GHashTable *node_hash, crm_time_t *now); - gboolean test_rule(xmlNode *rule, GHashTable *node_hash, enum rsc_role_e role, --- -1.8.3.1 - - -From 56a1337a54f3ba8a175ff3252658e1e43f7c670b Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Tue, 28 Apr 2020 14:34:40 -0400 -Subject: [PATCH 03/17] Feature: scheduler: Add new rule tests for op_defaults - and rsc_defaults. - -These are like all the other rule evaluating functions, but they do not -have any wrappers for the older style API. ---- - include/crm/pengine/rules_internal.h | 2 ++ - lib/pengine/rules.c | 68 ++++++++++++++++++++++++++++++++++++ - 2 files changed, 70 insertions(+) - -diff --git a/include/crm/pengine/rules_internal.h b/include/crm/pengine/rules_internal.h -index 8a22108..f60263a 100644 ---- a/include/crm/pengine/rules_internal.h -+++ b/include/crm/pengine/rules_internal.h -@@ -24,7 +24,9 @@ crm_time_t *pe_parse_xml_duration(crm_time_t * start, xmlNode * duration_spec); - gboolean pe__eval_attr_expr(xmlNode *expr, pe_rule_eval_data_t *rule_data); - int pe__eval_date_expr(xmlNode *expr, pe_rule_eval_data_t *rule_data, - crm_time_t *next_change); -+gboolean pe__eval_op_expr(xmlNodePtr expr, pe_rule_eval_data_t *rule_data); - gboolean pe__eval_role_expr(xmlNode *expr, pe_rule_eval_data_t *rule_data); -+gboolean pe__eval_rsc_expr(xmlNodePtr expr, pe_rule_eval_data_t *rule_data); - - int pe_eval_date_expression(xmlNode *time_expr, - crm_time_t *now, -diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c -index 3f316c2..a5af57a 100644 ---- a/lib/pengine/rules.c -+++ b/lib/pengine/rules.c -@@ -1123,6 +1123,38 @@ pe__eval_date_expr(xmlNodePtr expr, pe_rule_eval_data_t *rule_data, crm_time_t * - } - - gboolean -+pe__eval_op_expr(xmlNodePtr expr, pe_rule_eval_data_t *rule_data) { -+ const char *name = crm_element_value(expr, XML_NVPAIR_ATTR_NAME); -+ const char *interval_s = crm_element_value(expr, XML_LRM_ATTR_INTERVAL); -+ guint interval; -+ -+ crm_trace("Testing op_defaults expression: %s", ID(expr)); -+ -+ if (rule_data->op_data == NULL) { -+ crm_trace("No operations data provided"); -+ return FALSE; -+ } -+ -+ interval = crm_parse_interval_spec(interval_s); -+ if (interval == 0 && errno != 0) { -+ crm_trace("Could not parse interval: %s", interval_s); -+ return FALSE; -+ } -+ -+ if (interval_s != NULL && interval != rule_data->op_data->interval) { -+ crm_trace("Interval doesn't match: %d != %d", interval, rule_data->op_data->interval); -+ return FALSE; -+ } -+ -+ if (!crm_str_eq(name, rule_data->op_data->op_name, TRUE)) { -+ crm_trace("Name doesn't match: %s != %s", name, rule_data->op_data->op_name); -+ return FALSE; -+ } -+ -+ return TRUE; -+} -+ -+gboolean - pe__eval_role_expr(xmlNodePtr expr, pe_rule_eval_data_t *rule_data) - { - gboolean accept = FALSE; -@@ -1163,6 +1195,42 @@ pe__eval_role_expr(xmlNodePtr expr, pe_rule_eval_data_t *rule_data) - return accept; - } - -+gboolean -+pe__eval_rsc_expr(xmlNodePtr expr, pe_rule_eval_data_t *rule_data) -+{ -+ const char *class = crm_element_value(expr, XML_AGENT_ATTR_CLASS); -+ const char *provider = crm_element_value(expr, XML_AGENT_ATTR_PROVIDER); -+ const char *type = crm_element_value(expr, XML_EXPR_ATTR_TYPE); -+ -+ crm_trace("Testing rsc_defaults expression: %s", ID(expr)); -+ -+ if (rule_data->rsc_data == NULL) { -+ crm_trace("No resource data provided"); -+ return FALSE; -+ } -+ -+ if (class != NULL && -+ !crm_str_eq(class, rule_data->rsc_data->standard, TRUE)) { -+ crm_trace("Class doesn't match: %s != %s", class, rule_data->rsc_data->standard); -+ return FALSE; -+ } -+ -+ if ((provider == NULL && rule_data->rsc_data->provider != NULL) || -+ (provider != NULL && rule_data->rsc_data->provider == NULL) || -+ !crm_str_eq(provider, rule_data->rsc_data->provider, TRUE)) { -+ crm_trace("Provider doesn't match: %s != %s", provider, rule_data->rsc_data->provider); -+ return FALSE; -+ } -+ -+ if (type != NULL && -+ !crm_str_eq(type, rule_data->rsc_data->agent, TRUE)) { -+ crm_trace("Agent doesn't match: %s != %s", type, rule_data->rsc_data->agent); -+ return FALSE; -+ } -+ -+ return TRUE; -+} -+ - // Deprecated functions kept only for backward API compatibility - gboolean test_ruleset(xmlNode *ruleset, GHashTable *node_hash, crm_time_t *now); - gboolean test_rule(xmlNode *rule, GHashTable *node_hash, enum rsc_role_e role, --- -1.8.3.1 - - -From 5a4da3f77feee0d3bac50e9adc4eb4b35724dfb2 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Tue, 28 Apr 2020 14:41:08 -0400 -Subject: [PATCH 04/17] Refactor: scheduler: Reimplement core rule eval - functions. - -The core functions of pe_evaluate_rules, pe_test_rule, and -pe_test_expression have been turned into new, similarly named functions -that take a pe_rule_eval_data_t as an argument. The old ones still -exist as wrappers around the new ones. ---- - include/crm/pengine/rules.h | 7 ++ - lib/pengine/rules.c | 259 ++++++++++++++++++++++++++------------------ - 2 files changed, 162 insertions(+), 104 deletions(-) - -diff --git a/include/crm/pengine/rules.h b/include/crm/pengine/rules.h -index d7bdbf9..a74c629 100644 ---- a/include/crm/pengine/rules.h -+++ b/include/crm/pengine/rules.h -@@ -61,6 +61,13 @@ GHashTable *pe_unpack_versioned_parameters(xmlNode *versioned_params, const char - - char *pe_expand_re_matches(const char *string, pe_re_match_data_t * match_data); - -+gboolean pe_eval_rules(xmlNode *ruleset, pe_rule_eval_data_t *rule_data, -+ crm_time_t *next_change); -+gboolean pe_eval_expr(xmlNode *rule, pe_rule_eval_data_t *rule_data, -+ crm_time_t *next_change); -+gboolean pe_eval_subexpr(xmlNode *expr, pe_rule_eval_data_t *rule_data, -+ crm_time_t *next_change); -+ - #ifndef PCMK__NO_COMPAT - /* Everything here is deprecated and kept only for public API backward - * compatibility. It will be moved to compatibility.h when 2.1.0 is released. -diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c -index a5af57a..a6353ef 100644 ---- a/lib/pengine/rules.c -+++ b/lib/pengine/rules.c -@@ -38,25 +38,16 @@ gboolean - pe_evaluate_rules(xmlNode *ruleset, GHashTable *node_hash, crm_time_t *now, - crm_time_t *next_change) - { -- // If there are no rules, pass by default -- gboolean ruleset_default = TRUE; -- -- for (xmlNode *rule = first_named_child(ruleset, XML_TAG_RULE); -- rule != NULL; rule = crm_next_same_xml(rule)) { -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = node_hash, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; - -- ruleset_default = FALSE; -- if (pe_test_rule(rule, node_hash, RSC_ROLE_UNKNOWN, now, next_change, -- NULL)) { -- /* Only the deprecated "lifetime" element of location constraints -- * may contain more than one rule at the top level -- the schema -- * limits a block of nvpairs to a single top-level rule. So, this -- * effectively means that a lifetime is active if any rule it -- * contains is active. -- */ -- return TRUE; -- } -- } -- return ruleset_default; -+ return pe_eval_rules(ruleset, &rule_data, next_change); - } - - gboolean -@@ -64,44 +55,16 @@ pe_test_rule(xmlNode *rule, GHashTable *node_hash, enum rsc_role_e role, - crm_time_t *now, crm_time_t *next_change, - pe_match_data_t *match_data) - { -- xmlNode *expr = NULL; -- gboolean test = TRUE; -- gboolean empty = TRUE; -- gboolean passed = TRUE; -- gboolean do_and = TRUE; -- const char *value = NULL; -- -- rule = expand_idref(rule, NULL); -- value = crm_element_value(rule, XML_RULE_ATTR_BOOLEAN_OP); -- if (safe_str_eq(value, "or")) { -- do_and = FALSE; -- passed = FALSE; -- } -- -- crm_trace("Testing rule %s", ID(rule)); -- for (expr = __xml_first_child_element(rule); expr != NULL; -- expr = __xml_next_element(expr)) { -- -- test = pe_test_expression(expr, node_hash, role, now, next_change, -- match_data); -- empty = FALSE; -- -- if (test && do_and == FALSE) { -- crm_trace("Expression %s/%s passed", ID(rule), ID(expr)); -- return TRUE; -- -- } else if (test == FALSE && do_and) { -- crm_trace("Expression %s/%s failed", ID(rule), ID(expr)); -- return FALSE; -- } -- } -- -- if (empty) { -- crm_err("Invalid Rule %s: rules must contain at least one expression", ID(rule)); -- } -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = node_hash, -+ .role = role, -+ .now = now, -+ .match_data = match_data, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; - -- crm_trace("Rule %s %s", ID(rule), passed ? "passed" : "failed"); -- return passed; -+ return pe_eval_expr(rule, &rule_data, next_change); - } - - /*! -@@ -125,56 +88,16 @@ pe_test_expression(xmlNode *expr, GHashTable *node_hash, enum rsc_role_e role, - crm_time_t *now, crm_time_t *next_change, - pe_match_data_t *match_data) - { -- gboolean accept = FALSE; -- const char *uname = NULL; -- -- switch (find_expression_type(expr)) { -- case nested_rule: -- accept = pe_test_rule(expr, node_hash, role, now, next_change, -- match_data); -- break; -- case attr_expr: -- case loc_expr: -- /* these expressions can never succeed if there is -- * no node to compare with -- */ -- if (node_hash != NULL) { -- accept = pe_test_attr_expression(expr, node_hash, now, match_data); -- } -- break; -- -- case time_expr: -- accept = pe_test_date_expression(expr, now, next_change); -- break; -- -- case role_expr: -- accept = pe_test_role_expression(expr, role, now); -- break; -- --#if ENABLE_VERSIONED_ATTRS -- case version_expr: -- if (node_hash && g_hash_table_lookup_extended(node_hash, -- CRM_ATTR_RA_VERSION, -- NULL, NULL)) { -- accept = pe_test_attr_expression(expr, node_hash, now, NULL); -- } else { -- // we are going to test it when we have ra-version -- accept = TRUE; -- } -- break; --#endif -- -- default: -- CRM_CHECK(FALSE /* bad type */ , return FALSE); -- accept = FALSE; -- } -- if (node_hash) { -- uname = g_hash_table_lookup(node_hash, CRM_ATTR_UNAME); -- } -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = node_hash, -+ .role = role, -+ .now = now, -+ .match_data = match_data, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; - -- crm_trace("Expression %s %s on %s", -- ID(expr), accept ? "passed" : "failed", uname ? uname : "all nodes"); -- return accept; -+ return pe_eval_subexpr(expr, &rule_data, next_change); - } - - enum expression_type -@@ -885,6 +808,134 @@ pe_unpack_versioned_parameters(xmlNode *versioned_params, const char *ra_version - #endif - - gboolean -+pe_eval_rules(xmlNode *ruleset, pe_rule_eval_data_t *rule_data, crm_time_t *next_change) -+{ -+ // If there are no rules, pass by default -+ gboolean ruleset_default = TRUE; -+ -+ for (xmlNode *rule = first_named_child(ruleset, XML_TAG_RULE); -+ rule != NULL; rule = crm_next_same_xml(rule)) { -+ -+ ruleset_default = FALSE; -+ if (pe_eval_expr(rule, rule_data, next_change)) { -+ /* Only the deprecated "lifetime" element of location constraints -+ * may contain more than one rule at the top level -- the schema -+ * limits a block of nvpairs to a single top-level rule. So, this -+ * effectively means that a lifetime is active if any rule it -+ * contains is active. -+ */ -+ return TRUE; -+ } -+ } -+ -+ return ruleset_default; -+} -+ -+gboolean -+pe_eval_expr(xmlNode *rule, pe_rule_eval_data_t *rule_data, crm_time_t *next_change) -+{ -+ xmlNode *expr = NULL; -+ gboolean test = TRUE; -+ gboolean empty = TRUE; -+ gboolean passed = TRUE; -+ gboolean do_and = TRUE; -+ const char *value = NULL; -+ -+ rule = expand_idref(rule, NULL); -+ value = crm_element_value(rule, XML_RULE_ATTR_BOOLEAN_OP); -+ if (safe_str_eq(value, "or")) { -+ do_and = FALSE; -+ passed = FALSE; -+ } -+ -+ crm_trace("Testing rule %s", ID(rule)); -+ for (expr = __xml_first_child_element(rule); expr != NULL; -+ expr = __xml_next_element(expr)) { -+ -+ test = pe_eval_subexpr(expr, rule_data, next_change); -+ empty = FALSE; -+ -+ if (test && do_and == FALSE) { -+ crm_trace("Expression %s/%s passed", ID(rule), ID(expr)); -+ return TRUE; -+ -+ } else if (test == FALSE && do_and) { -+ crm_trace("Expression %s/%s failed", ID(rule), ID(expr)); -+ return FALSE; -+ } -+ } -+ -+ if (empty) { -+ crm_err("Invalid Rule %s: rules must contain at least one expression", ID(rule)); -+ } -+ -+ crm_trace("Rule %s %s", ID(rule), passed ? "passed" : "failed"); -+ return passed; -+} -+ -+gboolean -+pe_eval_subexpr(xmlNode *expr, pe_rule_eval_data_t *rule_data, crm_time_t *next_change) -+{ -+ gboolean accept = FALSE; -+ const char *uname = NULL; -+ -+ switch (find_expression_type(expr)) { -+ case nested_rule: -+ accept = pe_eval_expr(expr, rule_data, next_change); -+ break; -+ case attr_expr: -+ case loc_expr: -+ /* these expressions can never succeed if there is -+ * no node to compare with -+ */ -+ if (rule_data->node_hash != NULL) { -+ accept = pe__eval_attr_expr(expr, rule_data); -+ } -+ break; -+ -+ case time_expr: -+ accept = pe_test_date_expression(expr, rule_data->now, next_change); -+ break; -+ -+ case role_expr: -+ accept = pe__eval_role_expr(expr, rule_data); -+ break; -+ -+ case rsc_expr: -+ accept = pe__eval_rsc_expr(expr, rule_data); -+ break; -+ -+ case op_expr: -+ accept = pe__eval_op_expr(expr, rule_data); -+ break; -+ -+#if ENABLE_VERSIONED_ATTRS -+ case version_expr: -+ if (rule_data->node_hash && -+ g_hash_table_lookup_extended(rule_data->node_hash, -+ CRM_ATTR_RA_VERSION, NULL, NULL)) { -+ accept = pe__eval_attr_expr(expr, rule_data); -+ } else { -+ // we are going to test it when we have ra-version -+ accept = TRUE; -+ } -+ break; -+#endif -+ -+ default: -+ CRM_CHECK(FALSE /* bad type */ , return FALSE); -+ accept = FALSE; -+ } -+ if (rule_data->node_hash) { -+ uname = g_hash_table_lookup(rule_data->node_hash, CRM_ATTR_UNAME); -+ } -+ -+ crm_trace("Expression %s %s on %s", -+ ID(expr), accept ? "passed" : "failed", uname ? uname : "all nodes"); -+ return accept; -+} -+ -+gboolean - pe__eval_attr_expr(xmlNodePtr expr, pe_rule_eval_data_t *rule_data) - { - gboolean accept = FALSE; --- -1.8.3.1 - - -From ea6318252164578fd27dcef657e80f5225337a4b Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Tue, 7 Apr 2020 15:57:06 -0400 -Subject: [PATCH 05/17] Refactor: scheduler: Add rule_data to unpack_data_s. - -This is just to get rid of a couple extra arguments to some internal -functions and make them look like the external functions. ---- - lib/pengine/rules.c | 65 ++++++++++++++++++++++++++++++++++++----------------- - 1 file changed, 44 insertions(+), 21 deletions(-) - -diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c -index a6353ef..2709d68 100644 ---- a/lib/pengine/rules.c -+++ b/lib/pengine/rules.c -@@ -555,10 +555,9 @@ add_versioned_attributes(xmlNode * attr_set, xmlNode * versioned_attrs) - - typedef struct unpack_data_s { - gboolean overwrite; -- GHashTable *node_hash; - void *hash; -- crm_time_t *now; - crm_time_t *next_change; -+ pe_rule_eval_data_t *rule_data; - xmlNode *top; - } unpack_data_t; - -@@ -568,14 +567,14 @@ unpack_attr_set(gpointer data, gpointer user_data) - sorted_set_t *pair = data; - unpack_data_t *unpack_data = user_data; - -- if (!pe_evaluate_rules(pair->attr_set, unpack_data->node_hash, -- unpack_data->now, unpack_data->next_change)) { -+ if (!pe_eval_rules(pair->attr_set, unpack_data->rule_data, -+ unpack_data->next_change)) { - return; - } - - #if ENABLE_VERSIONED_ATTRS -- if (get_versioned_rule(pair->attr_set) && !(unpack_data->node_hash && -- g_hash_table_lookup_extended(unpack_data->node_hash, -+ if (get_versioned_rule(pair->attr_set) && !(unpack_data->rule_data->node_hash && -+ g_hash_table_lookup_extended(unpack_data->rule_data->node_hash, - CRM_ATTR_RA_VERSION, NULL, NULL))) { - // we haven't actually tested versioned expressions yet - return; -@@ -593,8 +592,8 @@ unpack_versioned_attr_set(gpointer data, gpointer user_data) - sorted_set_t *pair = data; - unpack_data_t *unpack_data = user_data; - -- if (pe_evaluate_rules(pair->attr_set, unpack_data->node_hash, -- unpack_data->now, unpack_data->next_change)) { -+ if (pe_eval_rules(pair->attr_set, unpack_data->rule_data, -+ unpack_data->next_change)) { - add_versioned_attributes(pair->attr_set, unpack_data->hash); - } - } -@@ -658,19 +657,17 @@ make_pairs(xmlNode *top, xmlNode *xml_obj, const char *set_name, - * \param[in] top XML document root (used to expand id-ref's) - * \param[in] xml_obj XML element containing blocks of nvpair elements - * \param[in] set_name If not NULL, only use blocks of this element type -- * \param[in] node_hash Node attributes to use when evaluating rules - * \param[out] hash Where to store extracted name/value pairs - * \param[in] always_first If not NULL, process block with this ID first - * \param[in] overwrite Whether to replace existing values with same name -- * \param[in] now Time to use when evaluating rules -+ * \param[in] rule_data Matching parameters to use when unpacking - * \param[out] next_change If not NULL, set to when rule evaluation will change - * \param[in] unpack_func Function to call to unpack each block - */ - static void - unpack_nvpair_blocks(xmlNode *top, xmlNode *xml_obj, const char *set_name, -- GHashTable *node_hash, void *hash, -- const char *always_first, gboolean overwrite, -- crm_time_t *now, crm_time_t *next_change, -+ void *hash, const char *always_first, gboolean overwrite, -+ pe_rule_eval_data_t *rule_data, crm_time_t *next_change, - GFunc unpack_func) - { - GList *pairs = make_pairs(top, xml_obj, set_name, always_first); -@@ -678,11 +675,10 @@ unpack_nvpair_blocks(xmlNode *top, xmlNode *xml_obj, const char *set_name, - if (pairs) { - unpack_data_t data = { - .hash = hash, -- .node_hash = node_hash, -- .now = now, - .overwrite = overwrite, - .next_change = next_change, - .top = top, -+ .rule_data = rule_data - }; - - g_list_foreach(pairs, unpack_func, &data); -@@ -709,8 +705,17 @@ pe_unpack_nvpairs(xmlNode *top, xmlNode *xml_obj, const char *set_name, - const char *always_first, gboolean overwrite, - crm_time_t *now, crm_time_t *next_change) - { -- unpack_nvpair_blocks(top, xml_obj, set_name, node_hash, hash, always_first, -- overwrite, now, next_change, unpack_attr_set); -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = node_hash, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ -+ unpack_nvpair_blocks(top, xml_obj, set_name, hash, always_first, -+ overwrite, &rule_data, next_change, unpack_attr_set); - } - - #if ENABLE_VERSIONED_ATTRS -@@ -720,8 +725,17 @@ pe_unpack_versioned_attributes(xmlNode *top, xmlNode *xml_obj, - xmlNode *hash, crm_time_t *now, - crm_time_t *next_change) - { -- unpack_nvpair_blocks(top, xml_obj, set_name, node_hash, hash, NULL, FALSE, -- now, next_change, unpack_versioned_attr_set); -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = node_hash, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ -+ unpack_nvpair_blocks(top, xml_obj, set_name, hash, NULL, FALSE, -+ &rule_data, next_change, unpack_versioned_attr_set); - } - #endif - -@@ -1366,6 +1380,15 @@ unpack_instance_attributes(xmlNode *top, xmlNode *xml_obj, const char *set_name, - const char *always_first, gboolean overwrite, - crm_time_t *now) - { -- unpack_nvpair_blocks(top, xml_obj, set_name, node_hash, hash, always_first, -- overwrite, now, NULL, unpack_attr_set); -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = node_hash, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ -+ unpack_nvpair_blocks(top, xml_obj, set_name, hash, always_first, -+ overwrite, &rule_data, NULL, unpack_attr_set); - } --- -1.8.3.1 - - -From 54646db6f5e4f1bb141b35798bcad5c3cc025afe Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Wed, 8 Apr 2020 10:41:41 -0400 -Subject: [PATCH 06/17] Refactor: scheduler: Change args to - pe__unpack_dataset_nvpairs. - -It should now take a pe_rule_eval_data_t instead of various separate -arguments. This will allow passing further data that needs to be tested -against in the future (such as rsc_defaults and op_defaults). It's also -convenient to make versions of pe_unpack_nvpairs and -pe_unpack_versioned_attributes that take the same arguments. - -Then, adapt callers of pe__unpack_dataset_nvpairs to pass the new -argument. ---- - include/crm/pengine/internal.h | 2 +- - include/crm/pengine/rules.h | 9 +++++++ - lib/pengine/complex.c | 41 ++++++++++++++++++++++------- - lib/pengine/rules.c | 23 ++++++++++++++-- - lib/pengine/unpack.c | 33 ++++++++++++++++++++--- - lib/pengine/utils.c | 60 +++++++++++++++++++++++++++++++----------- - 6 files changed, 137 insertions(+), 31 deletions(-) - -diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h -index 189ba7b..3e59502 100644 ---- a/include/crm/pengine/internal.h -+++ b/include/crm/pengine/internal.h -@@ -460,7 +460,7 @@ void pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set); - void pe__register_messages(pcmk__output_t *out); - - void pe__unpack_dataset_nvpairs(xmlNode *xml_obj, const char *set_name, -- GHashTable *node_hash, GHashTable *hash, -+ pe_rule_eval_data_t *rule_data, GHashTable *hash, - const char *always_first, gboolean overwrite, - pe_working_set_t *data_set); - -diff --git a/include/crm/pengine/rules.h b/include/crm/pengine/rules.h -index a74c629..cbae8ed 100644 ---- a/include/crm/pengine/rules.h -+++ b/include/crm/pengine/rules.h -@@ -46,12 +46,21 @@ gboolean pe_test_expression(xmlNode *expr, GHashTable *node_hash, - crm_time_t *next_change, - pe_match_data_t *match_data); - -+void pe_eval_nvpairs(xmlNode *top, xmlNode *xml_obj, const char *set_name, -+ pe_rule_eval_data_t *rule_data, GHashTable *hash, -+ const char *always_first, gboolean overwrite, -+ crm_time_t *next_change); -+ - void pe_unpack_nvpairs(xmlNode *top, xmlNode *xml_obj, const char *set_name, - GHashTable *node_hash, GHashTable *hash, - const char *always_first, gboolean overwrite, - crm_time_t *now, crm_time_t *next_change); - - #if ENABLE_VERSIONED_ATTRS -+void pe_eval_versioned_attributes(xmlNode *top, xmlNode *xml_obj, -+ const char *set_name, pe_rule_eval_data_t *rule_data, -+ xmlNode *hash, crm_time_t *next_change); -+ - void pe_unpack_versioned_attributes(xmlNode *top, xmlNode *xml_obj, - const char *set_name, GHashTable *node_hash, - xmlNode *hash, crm_time_t *now, -diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c -index 16f3a71..d91c95e 100644 ---- a/lib/pengine/complex.c -+++ b/lib/pengine/complex.c -@@ -95,10 +95,17 @@ void - get_meta_attributes(GHashTable * meta_hash, pe_resource_t * rsc, - pe_node_t * node, pe_working_set_t * data_set) - { -- GHashTable *node_hash = NULL; -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = NULL, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = data_set->now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; - - if (node) { -- node_hash = node->details->attrs; -+ rule_data.node_hash = node->details->attrs; - } - - if (rsc->xml) { -@@ -112,7 +119,7 @@ get_meta_attributes(GHashTable * meta_hash, pe_resource_t * rsc, - } - } - -- pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_META_SETS, node_hash, -+ pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_META_SETS, &rule_data, - meta_hash, NULL, FALSE, data_set); - - /* set anything else based on the parent */ -@@ -122,20 +129,27 @@ get_meta_attributes(GHashTable * meta_hash, pe_resource_t * rsc, - - /* and finally check the defaults */ - pe__unpack_dataset_nvpairs(data_set->rsc_defaults, XML_TAG_META_SETS, -- node_hash, meta_hash, NULL, FALSE, data_set); -+ &rule_data, meta_hash, NULL, FALSE, data_set); - } - - void - get_rsc_attributes(GHashTable * meta_hash, pe_resource_t * rsc, - pe_node_t * node, pe_working_set_t * data_set) - { -- GHashTable *node_hash = NULL; -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = NULL, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = data_set->now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; - - if (node) { -- node_hash = node->details->attrs; -+ rule_data.node_hash = node->details->attrs; - } - -- pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_ATTR_SETS, node_hash, -+ pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_ATTR_SETS, &rule_data, - meta_hash, NULL, FALSE, data_set); - - /* set anything else based on the parent */ -@@ -145,7 +159,7 @@ get_rsc_attributes(GHashTable * meta_hash, pe_resource_t * rsc, - } else { - /* and finally check the defaults */ - pe__unpack_dataset_nvpairs(data_set->rsc_defaults, XML_TAG_ATTR_SETS, -- node_hash, meta_hash, NULL, FALSE, data_set); -+ &rule_data, meta_hash, NULL, FALSE, data_set); - } - } - -@@ -376,6 +390,15 @@ common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc, - bool remote_node = FALSE; - bool has_versioned_params = FALSE; - -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = NULL, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = data_set->now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ - crm_log_xml_trace(xml_obj, "Processing resource input..."); - - if (id == NULL) { -@@ -706,7 +729,7 @@ common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc, - - (*rsc)->utilization = crm_str_table_new(); - -- pe__unpack_dataset_nvpairs((*rsc)->xml, XML_TAG_UTILIZATION, NULL, -+ pe__unpack_dataset_nvpairs((*rsc)->xml, XML_TAG_UTILIZATION, &rule_data, - (*rsc)->utilization, NULL, FALSE, data_set); - - /* data_set->resources = g_list_append(data_set->resources, (*rsc)); */ -diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c -index 2709d68..7575011 100644 ---- a/lib/pengine/rules.c -+++ b/lib/pengine/rules.c -@@ -686,6 +686,16 @@ unpack_nvpair_blocks(xmlNode *top, xmlNode *xml_obj, const char *set_name, - } - } - -+void -+pe_eval_nvpairs(xmlNode *top, xmlNode *xml_obj, const char *set_name, -+ pe_rule_eval_data_t *rule_data, GHashTable *hash, -+ const char *always_first, gboolean overwrite, -+ crm_time_t *next_change) -+{ -+ unpack_nvpair_blocks(top, xml_obj, set_name, hash, always_first, -+ overwrite, rule_data, next_change, unpack_attr_set); -+} -+ - /*! - * \brief Extract nvpair blocks contained by an XML element into a hash table - * -@@ -714,12 +724,21 @@ pe_unpack_nvpairs(xmlNode *top, xmlNode *xml_obj, const char *set_name, - .op_data = NULL - }; - -- unpack_nvpair_blocks(top, xml_obj, set_name, hash, always_first, -- overwrite, &rule_data, next_change, unpack_attr_set); -+ pe_eval_nvpairs(top, xml_obj, set_name, &rule_data, hash, -+ always_first, overwrite, next_change); - } - - #if ENABLE_VERSIONED_ATTRS - void -+pe_eval_versioned_attributes(xmlNode *top, xmlNode *xml_obj, const char *set_name, -+ pe_rule_eval_data_t *rule_data, xmlNode *hash, -+ crm_time_t *next_change) -+{ -+ unpack_nvpair_blocks(top, xml_obj, set_name, hash, NULL, FALSE, rule_data, -+ next_change, unpack_versioned_attr_set); -+} -+ -+void - pe_unpack_versioned_attributes(xmlNode *top, xmlNode *xml_obj, - const char *set_name, GHashTable *node_hash, - xmlNode *hash, crm_time_t *now, -diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c -index 532a3e6..8784857 100644 ---- a/lib/pengine/unpack.c -+++ b/lib/pengine/unpack.c -@@ -188,9 +188,18 @@ unpack_config(xmlNode * config, pe_working_set_t * data_set) - const char *value = NULL; - GHashTable *config_hash = crm_str_table_new(); - -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = NULL, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = data_set->now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ - data_set->config_hash = config_hash; - -- pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, NULL, config_hash, -+ pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash, - CIB_OPTIONS_FIRST, FALSE, data_set); - - verify_pe_options(data_set->config_hash); -@@ -515,6 +524,15 @@ unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set) - const char *type = NULL; - const char *score = NULL; - -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = NULL, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = data_set->now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ - for (xml_obj = __xml_first_child_element(xml_nodes); xml_obj != NULL; - xml_obj = __xml_next_element(xml_obj)) { - -@@ -547,7 +565,7 @@ unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set) - handle_startup_fencing(data_set, new_node); - - add_node_attrs(xml_obj, new_node, FALSE, data_set); -- pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, NULL, -+ pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, &rule_data, - new_node->details->utilization, NULL, - FALSE, data_set); - -@@ -3698,6 +3716,15 @@ add_node_attrs(xmlNode *xml_obj, pe_node_t *node, bool overwrite, - { - const char *cluster_name = NULL; - -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = NULL, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = data_set->now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ - g_hash_table_insert(node->details->attrs, - strdup(CRM_ATTR_UNAME), strdup(node->details->uname)); - -@@ -3719,7 +3746,7 @@ add_node_attrs(xmlNode *xml_obj, pe_node_t *node, bool overwrite, - strdup(cluster_name)); - } - -- pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, NULL, -+ pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, &rule_data, - node->details->attrs, NULL, overwrite, data_set); - - if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) { -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index c9b45e0..d01936d 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -597,10 +597,19 @@ custom_action(pe_resource_t * rsc, char *key, const char *task, - - if (is_set(action->flags, pe_action_have_node_attrs) == FALSE - && action->node != NULL && action->op_entry != NULL) { -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = action->node->details->attrs, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = data_set->now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ - pe_set_action_bit(action, pe_action_have_node_attrs); - pe__unpack_dataset_nvpairs(action->op_entry, XML_TAG_ATTR_SETS, -- action->node->details->attrs, -- action->extra, NULL, FALSE, data_set); -+ &rule_data, action->extra, NULL, -+ FALSE, data_set); - } - - if (is_set(action->flags, pe_action_pseudo)) { -@@ -873,6 +882,15 @@ pe_get_configured_timeout(pe_resource_t *rsc, const char *action, pe_working_set - const char *timeout = NULL; - int timeout_ms = 0; - -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = NULL, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = data_set->now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ - for (child = first_named_child(rsc->ops_xml, XML_ATTR_OP); - child != NULL; child = crm_next_same_xml(child)) { - if (safe_str_eq(action, crm_element_value(child, XML_NVPAIR_ATTR_NAME))) { -@@ -884,7 +902,7 @@ pe_get_configured_timeout(pe_resource_t *rsc, const char *action, pe_working_set - if (timeout == NULL && data_set->op_defaults) { - GHashTable *action_meta = crm_str_table_new(); - pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS, -- NULL, action_meta, NULL, FALSE, data_set); -+ &rule_data, action_meta, NULL, FALSE, data_set); - timeout = g_hash_table_lookup(action_meta, XML_ATTR_TIMEOUT); - } - -@@ -964,10 +982,19 @@ unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * contai - pe_rsc_action_details_t *rsc_details = NULL; - #endif - -+ pe_rule_eval_data_t rule_data = { -+ .node_hash = NULL, -+ .role = RSC_ROLE_UNKNOWN, -+ .now = data_set->now, -+ .match_data = NULL, -+ .rsc_data = NULL, -+ .op_data = NULL -+ }; -+ - CRM_CHECK(action && action->rsc, return); - - // Cluster-wide -- pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS, NULL, -+ pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS, &rule_data, - action->meta, NULL, FALSE, data_set); - - // Probe timeouts default differently, so handle timeout default later -@@ -981,19 +1008,20 @@ unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * contai - xmlAttrPtr xIter = NULL; - - // take precedence over defaults -- pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_META_SETS, NULL, -+ pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_META_SETS, &rule_data, - action->meta, NULL, TRUE, data_set); - - #if ENABLE_VERSIONED_ATTRS - rsc_details = pe_rsc_action_details(action); -- pe_unpack_versioned_attributes(data_set->input, xml_obj, -- XML_TAG_ATTR_SETS, NULL, -- rsc_details->versioned_parameters, -- data_set->now, NULL); -- pe_unpack_versioned_attributes(data_set->input, xml_obj, -- XML_TAG_META_SETS, NULL, -- rsc_details->versioned_meta, -- data_set->now, NULL); -+ -+ pe_eval_versioned_attributes(data_set->input, xml_obj, -+ XML_TAG_ATTR_SETS, &rule_data, -+ rsc_details->versioned_parameters, -+ NULL); -+ pe_eval_versioned_attributes(data_set->input, xml_obj, -+ XML_TAG_META_SETS, &rule_data, -+ rsc_details->versioned_meta, -+ NULL); - #endif - - /* Anything set as an XML property has highest precedence. -@@ -2693,14 +2721,14 @@ pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set) - */ - void - pe__unpack_dataset_nvpairs(xmlNode *xml_obj, const char *set_name, -- GHashTable *node_hash, GHashTable *hash, -+ pe_rule_eval_data_t *rule_data, GHashTable *hash, - const char *always_first, gboolean overwrite, - pe_working_set_t *data_set) - { - crm_time_t *next_change = crm_time_new_undefined(); - -- pe_unpack_nvpairs(data_set->input, xml_obj, set_name, node_hash, hash, -- always_first, overwrite, data_set->now, next_change); -+ pe_eval_nvpairs(data_set->input, xml_obj, set_name, rule_data, hash, -+ always_first, overwrite, next_change); - if (crm_time_is_defined(next_change)) { - time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change); - --- -1.8.3.1 - - -From ad06f60bae1fcb5d204fa18a0b21ade78aaee5f4 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Wed, 8 Apr 2020 13:43:26 -0400 -Subject: [PATCH 07/17] Refactor: scheduler: unpack_operation should be static. - ---- - lib/pengine/utils.c | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index d01936d..c345875 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -23,8 +23,8 @@ - extern xmlNode *get_object_root(const char *object_type, xmlNode * the_root); - void print_str_str(gpointer key, gpointer value, gpointer user_data); - gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data); --void unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container, -- pe_working_set_t * data_set); -+static void unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container, -+ pe_working_set_t * data_set); - static xmlNode *find_rsc_op_entry_helper(pe_resource_t * rsc, const char *key, - gboolean include_disabled); - -@@ -968,7 +968,7 @@ unpack_versioned_meta(xmlNode *versioned_meta, xmlNode *xml_obj, - * \param[in] container Resource that contains affected resource, if any - * \param[in] data_set Cluster state - */ --void -+static void - unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container, - pe_working_set_t * data_set) - { --- -1.8.3.1 - - -From 7e57d955c9209af62dffc0639c50d51121028c26 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Wed, 8 Apr 2020 14:58:35 -0400 -Subject: [PATCH 08/17] Refactor: scheduler: Pass interval to unpack_operation. - ---- - lib/pengine/utils.c | 36 ++++++++++++++---------------------- - 1 file changed, 14 insertions(+), 22 deletions(-) - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index c345875..1e3b0bd 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -24,7 +24,7 @@ extern xmlNode *get_object_root(const char *object_type, xmlNode * the_root); - void print_str_str(gpointer key, gpointer value, gpointer user_data); - gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data); - static void unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container, -- pe_working_set_t * data_set); -+ pe_working_set_t * data_set, guint interval_ms); - static xmlNode *find_rsc_op_entry_helper(pe_resource_t * rsc, const char *key, - gboolean include_disabled); - -@@ -568,9 +568,13 @@ custom_action(pe_resource_t * rsc, char *key, const char *task, - } - - if (rsc != NULL) { -+ guint interval_ms = 0; -+ - action->op_entry = find_rsc_op_entry_helper(rsc, key, TRUE); -+ parse_op_key(key, NULL, NULL, &interval_ms); - -- unpack_operation(action, action->op_entry, rsc->container, data_set); -+ unpack_operation(action, action->op_entry, rsc->container, data_set, -+ interval_ms); - - if (save_action) { - rsc->actions = g_list_prepend(rsc->actions, action); -@@ -963,20 +967,20 @@ unpack_versioned_meta(xmlNode *versioned_meta, xmlNode *xml_obj, - * and start delay values as integer milliseconds), requirements, and - * failure policy. - * -- * \param[in,out] action Action to unpack into -- * \param[in] xml_obj Operation XML (or NULL if all defaults) -- * \param[in] container Resource that contains affected resource, if any -- * \param[in] data_set Cluster state -+ * \param[in,out] action Action to unpack into -+ * \param[in] xml_obj Operation XML (or NULL if all defaults) -+ * \param[in] container Resource that contains affected resource, if any -+ * \param[in] data_set Cluster state -+ * \param[in] interval_ms How frequently to perform the operation - */ - static void - unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container, -- pe_working_set_t * data_set) -+ pe_working_set_t * data_set, guint interval_ms) - { -- guint interval_ms = 0; - int timeout = 0; - char *value_ms = NULL; - const char *value = NULL; -- const char *field = NULL; -+ const char *field = XML_LRM_ATTR_INTERVAL; - char *default_timeout = NULL; - #if ENABLE_VERSIONED_ATTRS - pe_rsc_action_details_t *rsc_details = NULL; -@@ -1038,23 +1042,11 @@ unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * contai - g_hash_table_remove(action->meta, "id"); - - // Normalize interval to milliseconds -- field = XML_LRM_ATTR_INTERVAL; -- value = g_hash_table_lookup(action->meta, field); -- if (value != NULL) { -- interval_ms = crm_parse_interval_spec(value); -- -- } else if ((xml_obj == NULL) && !strcmp(action->task, RSC_STATUS)) { -- /* An orphaned recurring monitor will not have any XML. However, we -- * want the interval to be set, so the action can be properly detected -- * as a recurring monitor. Parse it from the key in this case. -- */ -- parse_op_key(action->uuid, NULL, NULL, &interval_ms); -- } - if (interval_ms > 0) { - value_ms = crm_strdup_printf("%u", interval_ms); - g_hash_table_replace(action->meta, strdup(field), value_ms); - -- } else if (value) { -+ } else if (g_hash_table_lookup(action->meta, field) != NULL) { - g_hash_table_remove(action->meta, field); - } - --- -1.8.3.1 - - -From e4c411d9674e222647dd3ed31714c369f54ccad1 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Thu, 9 Apr 2020 16:15:17 -0400 -Subject: [PATCH 09/17] Feature: scheduler: Pass rsc_defaults and op_defaults - data. - -See: rhbz#1628701. ---- - lib/pengine/complex.c | 8 +++++++- - lib/pengine/utils.c | 15 +++++++++++++-- - 2 files changed, 20 insertions(+), 3 deletions(-) - -diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c -index d91c95e..1f06348 100644 ---- a/lib/pengine/complex.c -+++ b/lib/pengine/complex.c -@@ -95,12 +95,18 @@ void - get_meta_attributes(GHashTable * meta_hash, pe_resource_t * rsc, - pe_node_t * node, pe_working_set_t * data_set) - { -+ pe_rsc_eval_data_t rsc_rule_data = { -+ .standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS), -+ .provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER), -+ .agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE) -+ }; -+ - pe_rule_eval_data_t rule_data = { - .node_hash = NULL, - .role = RSC_ROLE_UNKNOWN, - .now = data_set->now, - .match_data = NULL, -- .rsc_data = NULL, -+ .rsc_data = &rsc_rule_data, - .op_data = NULL - }; - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index 1e3b0bd..d5309ed 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -986,13 +986,24 @@ unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * contai - pe_rsc_action_details_t *rsc_details = NULL; - #endif - -+ pe_rsc_eval_data_t rsc_rule_data = { -+ .standard = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_CLASS), -+ .provider = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_PROVIDER), -+ .agent = crm_element_value(action->rsc->xml, XML_EXPR_ATTR_TYPE) -+ }; -+ -+ pe_op_eval_data_t op_rule_data = { -+ .op_name = action->task, -+ .interval = interval_ms -+ }; -+ - pe_rule_eval_data_t rule_data = { - .node_hash = NULL, - .role = RSC_ROLE_UNKNOWN, - .now = data_set->now, - .match_data = NULL, -- .rsc_data = NULL, -- .op_data = NULL -+ .rsc_data = &rsc_rule_data, -+ .op_data = &op_rule_data - }; - - CRM_CHECK(action && action->rsc, return); --- -1.8.3.1 - - -From 57eedcad739071530f01e1fd691734f7681a08a1 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Fri, 17 Apr 2020 12:30:51 -0400 -Subject: [PATCH 10/17] Feature: xml: Add rsc_expression and op_expression to - the XML schema. - ---- - cts/cli/regression.upgrade.exp | 7 +- - cts/cli/regression.validity.exp | 22 ++- - xml/constraints-next.rng | 4 +- - xml/nodes-3.4.rng | 44 +++++ - xml/nvset-3.4.rng | 63 ++++++ - xml/options-3.4.rng | 111 +++++++++++ - xml/resources-3.4.rng | 425 ++++++++++++++++++++++++++++++++++++++++ - xml/rule-3.4.rng | 165 ++++++++++++++++ - 8 files changed, 833 insertions(+), 8 deletions(-) - create mode 100644 xml/nodes-3.4.rng - create mode 100644 xml/nvset-3.4.rng - create mode 100644 xml/options-3.4.rng - create mode 100644 xml/resources-3.4.rng - create mode 100644 xml/rule-3.4.rng - -diff --git a/cts/cli/regression.upgrade.exp b/cts/cli/regression.upgrade.exp -index 28ca057..50b22df 100644 ---- a/cts/cli/regression.upgrade.exp -+++ b/cts/cli/regression.upgrade.exp -@@ -79,8 +79,11 @@ update_validation debug: Configuration valid for schema: pacemaker-3.2 - update_validation debug: pacemaker-3.2-style configuration is also valid for pacemaker-3.3 - update_validation debug: Testing 'pacemaker-3.3' validation (17 of X) - update_validation debug: Configuration valid for schema: pacemaker-3.3 --update_validation trace: Stopping at pacemaker-3.3 --update_validation info: Transformed the configuration from pacemaker-2.10 to pacemaker-3.3 -+update_validation debug: pacemaker-3.3-style configuration is also valid for pacemaker-3.4 -+update_validation debug: Testing 'pacemaker-3.4' validation (18 of X) -+update_validation debug: Configuration valid for schema: pacemaker-3.4 -+update_validation trace: Stopping at pacemaker-3.4 -+update_validation info: Transformed the configuration from pacemaker-2.10 to pacemaker-3.4 - =#=#=#= Current cib after: Upgrade to latest CIB schema (trigger 2.10.xsl + the wrapping) =#=#=#= - - -diff --git a/cts/cli/regression.validity.exp b/cts/cli/regression.validity.exp -index 46e54b5..4407074 100644 ---- a/cts/cli/regression.validity.exp -+++ b/cts/cli/regression.validity.exp -@@ -105,7 +105,11 @@ update_validation debug: Testing 'pacemaker-3.3' validation (17 of X) - element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order - element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order - update_validation trace: pacemaker-3.3 validation failed --Cannot upgrade configuration (claiming schema pacemaker-1.2) to at least pacemaker-3.0 because it does not validate with any schema from pacemaker-1.2 to pacemaker-3.3 -+update_validation debug: Testing 'pacemaker-3.4' validation (18 of X) -+element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order -+element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order -+update_validation trace: pacemaker-3.4 validation failed -+Cannot upgrade configuration (claiming schema pacemaker-1.2) to at least pacemaker-3.0 because it does not validate with any schema from pacemaker-1.2 to pacemaker-3.4 - =#=#=#= End test: Run crm_simulate with invalid CIB (enum violation) - Invalid configuration (78) =#=#=#= - * Passed: crm_simulate - Run crm_simulate with invalid CIB (enum violation) - =#=#=#= Begin test: Try to make resulting CIB invalid (unrecognized validate-with) =#=#=#= -@@ -198,7 +202,10 @@ update_validation trace: pacemaker-3.2 validation failed - update_validation debug: Testing 'pacemaker-3.3' validation (17 of X) - element cib: Relax-NG validity error : Invalid attribute validate-with for element cib - update_validation trace: pacemaker-3.3 validation failed --Cannot upgrade configuration (claiming schema pacemaker-9999.0) to at least pacemaker-3.0 because it does not validate with any schema from unknown to pacemaker-3.3 -+update_validation debug: Testing 'pacemaker-3.4' validation (18 of X) -+element cib: Relax-NG validity error : Invalid attribute validate-with for element cib -+update_validation trace: pacemaker-3.4 validation failed -+Cannot upgrade configuration (claiming schema pacemaker-9999.0) to at least pacemaker-3.0 because it does not validate with any schema from unknown to pacemaker-3.4 - =#=#=#= End test: Run crm_simulate with invalid CIB (unrecognized validate-with) - Invalid configuration (78) =#=#=#= - * Passed: crm_simulate - Run crm_simulate with invalid CIB (unrecognized validate-with) - =#=#=#= Begin test: Try to make resulting CIB invalid, but possibly recoverable (valid with X.Y+1) =#=#=#= -@@ -286,8 +293,11 @@ update_validation debug: Configuration valid for schema: pacemaker-3.2 - update_validation debug: pacemaker-3.2-style configuration is also valid for pacemaker-3.3 - update_validation debug: Testing 'pacemaker-3.3' validation (17 of X) - update_validation debug: Configuration valid for schema: pacemaker-3.3 --update_validation trace: Stopping at pacemaker-3.3 --update_validation info: Transformed the configuration from pacemaker-1.2 to pacemaker-3.3 -+update_validation debug: pacemaker-3.3-style configuration is also valid for pacemaker-3.4 -+update_validation debug: Testing 'pacemaker-3.4' validation (18 of X) -+update_validation debug: Configuration valid for schema: pacemaker-3.4 -+update_validation trace: Stopping at pacemaker-3.4 -+update_validation info: Transformed the configuration from pacemaker-1.2 to pacemaker-3.4 - unpack_resources error: Resource start-up disabled since no STONITH resources have been defined - unpack_resources error: Either configure some or disable STONITH with the stonith-enabled option - unpack_resources error: NOTE: Clusters with shared data need STONITH to ensure data integrity -@@ -393,6 +403,8 @@ element rsc_order: Relax-NG validity error : Invalid attribute first-action for - element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order - element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order - element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order -+element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order -+element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order - =#=#=#= Current cib after: Make resulting CIB invalid, and without validate-with attribute =#=#=#= - - -@@ -450,6 +462,8 @@ validity.bad.xml:10: element rsc_order: Relax-NG validity error : Invalid attrib - validity.bad.xml:10: element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order - validity.bad.xml:10: element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order - validity.bad.xml:10: element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order -+validity.bad.xml:10: element rsc_order: Relax-NG validity error : Invalid attribute first-action for element rsc_order -+validity.bad.xml:10: element rsc_order: Relax-NG validity error : Element constraints has extra content: rsc_order - unpack_resources error: Resource start-up disabled since no STONITH resources have been defined - unpack_resources error: Either configure some or disable STONITH with the stonith-enabled option - unpack_resources error: NOTE: Clusters with shared data need STONITH to ensure data integrity -diff --git a/xml/constraints-next.rng b/xml/constraints-next.rng -index 7e0d98e..1fa3e75 100644 ---- a/xml/constraints-next.rng -+++ b/xml/constraints-next.rng -@@ -43,7 +43,7 @@ - - - -- -+ - - - -@@ -255,7 +255,7 @@ - - - -- -+ - - - -diff --git a/xml/nodes-3.4.rng b/xml/nodes-3.4.rng -new file mode 100644 -index 0000000..0132c72 ---- /dev/null -+++ b/xml/nodes-3.4.rng -@@ -0,0 +1,44 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ member -+ ping -+ remote -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/xml/nvset-3.4.rng b/xml/nvset-3.4.rng -new file mode 100644 -index 0000000..91a7d23 ---- /dev/null -+++ b/xml/nvset-3.4.rng -@@ -0,0 +1,63 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/xml/options-3.4.rng b/xml/options-3.4.rng -new file mode 100644 -index 0000000..22330d8 ---- /dev/null -+++ b/xml/options-3.4.rng -@@ -0,0 +1,111 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ cluster-infrastructure -+ -+ -+ -+ -+ -+ heartbeat -+ openais -+ classic openais -+ classic openais (with plugin) -+ cman -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ cluster-infrastructure -+ cluster_recheck_interval -+ dc_deadtime -+ default-action-timeout -+ default_action_timeout -+ default-migration-threshold -+ default_migration_threshold -+ default-resource-failure-stickiness -+ default_resource_failure_stickiness -+ default-resource-stickiness -+ default_resource_stickiness -+ election_timeout -+ expected-quorum-votes -+ is-managed-default -+ is_managed_default -+ no_quorum_policy -+ notification-agent -+ notification-recipient -+ remove_after_stop -+ shutdown_escalation -+ startup_fencing -+ stonith_action -+ stonith_enabled -+ stop_orphan_actions -+ stop_orphan_resources -+ symmetric_cluster -+ transition_idle_timeout -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/xml/resources-3.4.rng b/xml/resources-3.4.rng -new file mode 100644 -index 0000000..fbb4b65 ---- /dev/null -+++ b/xml/resources-3.4.rng -@@ -0,0 +1,425 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ isolation -+ isolation-host -+ isolation-instance -+ isolation-wrapper -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ ([0-9\-]+) -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ requires -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ Stopped -+ Started -+ Slave -+ Master -+ -+ -+ -+ -+ -+ -+ ignore -+ block -+ stop -+ restart -+ standby -+ fence -+ restart-container -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ ocf -+ -+ -+ -+ -+ lsb -+ heartbeat -+ stonith -+ upstart -+ service -+ systemd -+ nagios -+ -+ -+ -+ -+ -diff --git a/xml/rule-3.4.rng b/xml/rule-3.4.rng -new file mode 100644 -index 0000000..5d1daf0 ---- /dev/null -+++ b/xml/rule-3.4.rng -@@ -0,0 +1,165 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ or -+ and -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ lt -+ gt -+ lte -+ gte -+ eq -+ ne -+ defined -+ not_defined -+ -+ -+ -+ -+ -+ -+ -+ -+ string -+ number -+ version -+ -+ -+ -+ -+ -+ -+ literal -+ param -+ meta -+ -+ -+ -+ -+ -+ -+ -+ -+ in_range -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ gt -+ -+ -+ -+ lt -+ -+ -+ -+ -+ -+ date_spec -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - - -From b0e2345d92fb7cf42c133b24457eeb07126db8a0 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Mon, 27 Apr 2020 16:24:22 -0400 -Subject: [PATCH 11/17] Fix: scheduler: Change trace output in populate_hash. - -Only show the "Setting attribute:" text when it comes time to actually -set the attribute. Also show the value being set. This makes it -clearer that an attribute is actually being set, not just that the -function is processing something. ---- - lib/pengine/rules.c | 3 +-- - 1 file changed, 1 insertion(+), 2 deletions(-) - -diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c -index 7575011..b0fca55 100644 ---- a/lib/pengine/rules.c -+++ b/lib/pengine/rules.c -@@ -463,7 +463,6 @@ populate_hash(xmlNode * nvpair_list, GHashTable * hash, gboolean overwrite, xmlN - name = crm_element_value(ref_nvpair, XML_NVPAIR_ATTR_NAME); - } - -- crm_trace("Setting attribute: %s", name); - value = crm_element_value(an_attr, XML_NVPAIR_ATTR_VALUE); - if (value == NULL) { - value = crm_element_value(ref_nvpair, XML_NVPAIR_ATTR_VALUE); -@@ -471,7 +470,6 @@ populate_hash(xmlNode * nvpair_list, GHashTable * hash, gboolean overwrite, xmlN - - if (name == NULL || value == NULL) { - continue; -- - } - - old_value = g_hash_table_lookup(hash, name); -@@ -484,6 +482,7 @@ populate_hash(xmlNode * nvpair_list, GHashTable * hash, gboolean overwrite, xmlN - continue; - - } else if (old_value == NULL) { -+ crm_trace("Setting attribute: %s = %s", name, value); - g_hash_table_insert(hash, strdup(name), strdup(value)); - - } else if (overwrite) { --- -1.8.3.1 - - -From d35854384b231c79b8aba1ce4c5caf5dd51ec982 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Fri, 1 May 2020 15:45:31 -0400 -Subject: [PATCH 12/17] Test: scheduler: Add a regression test for op_defaults. - ---- - cts/cts-scheduler.in | 3 + - cts/scheduler/op-defaults.dot | 33 ++++++ - cts/scheduler/op-defaults.exp | 211 ++++++++++++++++++++++++++++++++++++++ - cts/scheduler/op-defaults.scores | 11 ++ - cts/scheduler/op-defaults.summary | 46 +++++++++ - cts/scheduler/op-defaults.xml | 87 ++++++++++++++++ - 6 files changed, 391 insertions(+) - create mode 100644 cts/scheduler/op-defaults.dot - create mode 100644 cts/scheduler/op-defaults.exp - create mode 100644 cts/scheduler/op-defaults.scores - create mode 100644 cts/scheduler/op-defaults.summary - create mode 100644 cts/scheduler/op-defaults.xml - -diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in -index 5d72205..b83f812 100644 ---- a/cts/cts-scheduler.in -+++ b/cts/cts-scheduler.in -@@ -962,6 +962,9 @@ TESTS = [ - [ "shutdown-lock", "Ensure shutdown lock works properly" ], - [ "shutdown-lock-expiration", "Ensure shutdown lock expiration works properly" ], - ], -+ [ -+ [ "op-defaults", "Test op_defaults conditional expressions " ], -+ ], - - # @TODO: If pacemaker implements versioned attributes, uncomment these tests - #[ -diff --git a/cts/scheduler/op-defaults.dot b/cts/scheduler/op-defaults.dot -new file mode 100644 -index 0000000..5536c15 ---- /dev/null -+++ b/cts/scheduler/op-defaults.dot -@@ -0,0 +1,33 @@ -+ digraph "g" { -+"dummy-rsc_monitor_0 cluster01" -> "dummy-rsc_start_0 cluster02" [ style = bold] -+"dummy-rsc_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"dummy-rsc_monitor_0 cluster02" -> "dummy-rsc_start_0 cluster02" [ style = bold] -+"dummy-rsc_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"dummy-rsc_monitor_60000 cluster02" [ style=bold color="green" fontcolor="black"] -+"dummy-rsc_start_0 cluster02" -> "dummy-rsc_monitor_60000 cluster02" [ style = bold] -+"dummy-rsc_start_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"fencing_monitor_0 cluster01" -> "fencing_start_0 cluster01" [ style = bold] -+"fencing_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"fencing_monitor_0 cluster02" -> "fencing_start_0 cluster01" [ style = bold] -+"fencing_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"fencing_start_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ip-rsc2_monitor_0 cluster01" -> "ip-rsc2_start_0 cluster01" [ style = bold] -+"ip-rsc2_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ip-rsc2_monitor_0 cluster02" -> "ip-rsc2_start_0 cluster01" [ style = bold] -+"ip-rsc2_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"ip-rsc2_monitor_10000 cluster01" [ style=bold color="green" fontcolor="black"] -+"ip-rsc2_start_0 cluster01" -> "ip-rsc2_monitor_10000 cluster01" [ style = bold] -+"ip-rsc2_start_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ip-rsc_monitor_0 cluster01" -> "ip-rsc_start_0 cluster02" [ style = bold] -+"ip-rsc_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ip-rsc_monitor_0 cluster02" -> "ip-rsc_start_0 cluster02" [ style = bold] -+"ip-rsc_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"ip-rsc_monitor_20000 cluster02" [ style=bold color="green" fontcolor="black"] -+"ip-rsc_start_0 cluster02" -> "ip-rsc_monitor_20000 cluster02" [ style = bold] -+"ip-rsc_start_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"ping-rsc-ping_monitor_0 cluster01" -> "ping-rsc-ping_start_0 cluster01" [ style = bold] -+"ping-rsc-ping_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ping-rsc-ping_monitor_0 cluster02" -> "ping-rsc-ping_start_0 cluster01" [ style = bold] -+"ping-rsc-ping_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"ping-rsc-ping_start_0 cluster01" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/op-defaults.exp b/cts/scheduler/op-defaults.exp -new file mode 100644 -index 0000000..b81eacb ---- /dev/null -+++ b/cts/scheduler/op-defaults.exp -@@ -0,0 +1,211 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/op-defaults.scores b/cts/scheduler/op-defaults.scores -new file mode 100644 -index 0000000..1c622f0 ---- /dev/null -+++ b/cts/scheduler/op-defaults.scores -@@ -0,0 +1,11 @@ -+Allocation scores: -+pcmk__native_allocate: dummy-rsc allocation score on cluster01: 0 -+pcmk__native_allocate: dummy-rsc allocation score on cluster02: 0 -+pcmk__native_allocate: fencing allocation score on cluster01: 0 -+pcmk__native_allocate: fencing allocation score on cluster02: 0 -+pcmk__native_allocate: ip-rsc allocation score on cluster01: 0 -+pcmk__native_allocate: ip-rsc allocation score on cluster02: 0 -+pcmk__native_allocate: ip-rsc2 allocation score on cluster01: 0 -+pcmk__native_allocate: ip-rsc2 allocation score on cluster02: 0 -+pcmk__native_allocate: ping-rsc-ping allocation score on cluster01: 0 -+pcmk__native_allocate: ping-rsc-ping allocation score on cluster02: 0 -diff --git a/cts/scheduler/op-defaults.summary b/cts/scheduler/op-defaults.summary -new file mode 100644 -index 0000000..b580939 ---- /dev/null -+++ b/cts/scheduler/op-defaults.summary -@@ -0,0 +1,46 @@ -+ -+Current cluster status: -+Online: [ cluster01 cluster02 ] -+ -+ fencing (stonith:fence_xvm): Stopped -+ ip-rsc (ocf::heartbeat:IPaddr2): Stopped -+ ip-rsc2 (ocf::heartbeat:IPaddr2): Stopped -+ dummy-rsc (ocf::pacemaker:Dummy): Stopped -+ ping-rsc-ping (ocf::pacemaker:ping): Stopped -+ -+Transition Summary: -+ * Start fencing ( cluster01 ) -+ * Start ip-rsc ( cluster02 ) -+ * Start ip-rsc2 ( cluster01 ) -+ * Start dummy-rsc ( cluster02 ) -+ * Start ping-rsc-ping ( cluster01 ) -+ -+Executing cluster transition: -+ * Resource action: fencing monitor on cluster02 -+ * Resource action: fencing monitor on cluster01 -+ * Resource action: ip-rsc monitor on cluster02 -+ * Resource action: ip-rsc monitor on cluster01 -+ * Resource action: ip-rsc2 monitor on cluster02 -+ * Resource action: ip-rsc2 monitor on cluster01 -+ * Resource action: dummy-rsc monitor on cluster02 -+ * Resource action: dummy-rsc monitor on cluster01 -+ * Resource action: ping-rsc-ping monitor on cluster02 -+ * Resource action: ping-rsc-ping monitor on cluster01 -+ * Resource action: fencing start on cluster01 -+ * Resource action: ip-rsc start on cluster02 -+ * Resource action: ip-rsc2 start on cluster01 -+ * Resource action: dummy-rsc start on cluster02 -+ * Resource action: ping-rsc-ping start on cluster01 -+ * Resource action: ip-rsc monitor=20000 on cluster02 -+ * Resource action: ip-rsc2 monitor=10000 on cluster01 -+ * Resource action: dummy-rsc monitor=60000 on cluster02 -+ -+Revised cluster status: -+Online: [ cluster01 cluster02 ] -+ -+ fencing (stonith:fence_xvm): Started cluster01 -+ ip-rsc (ocf::heartbeat:IPaddr2): Started cluster02 -+ ip-rsc2 (ocf::heartbeat:IPaddr2): Started cluster01 -+ dummy-rsc (ocf::pacemaker:Dummy): Started cluster02 -+ ping-rsc-ping (ocf::pacemaker:ping): Started cluster01 -+ -diff --git a/cts/scheduler/op-defaults.xml b/cts/scheduler/op-defaults.xml -new file mode 100644 -index 0000000..ae3b248 ---- /dev/null -+++ b/cts/scheduler/op-defaults.xml -@@ -0,0 +1,87 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - - -From 67067927bc1b8e000c06d2b5a4ae6b9223ca13c7 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Wed, 13 May 2020 10:40:34 -0400 -Subject: [PATCH 13/17] Test: scheduler: Add a regression test for - rsc_defaults. - ---- - cts/cts-scheduler.in | 3 +- - cts/scheduler/rsc-defaults.dot | 18 ++++++ - cts/scheduler/rsc-defaults.exp | 124 +++++++++++++++++++++++++++++++++++++ - cts/scheduler/rsc-defaults.scores | 11 ++++ - cts/scheduler/rsc-defaults.summary | 38 ++++++++++++ - cts/scheduler/rsc-defaults.xml | 78 +++++++++++++++++++++++ - 6 files changed, 271 insertions(+), 1 deletion(-) - create mode 100644 cts/scheduler/rsc-defaults.dot - create mode 100644 cts/scheduler/rsc-defaults.exp - create mode 100644 cts/scheduler/rsc-defaults.scores - create mode 100644 cts/scheduler/rsc-defaults.summary - create mode 100644 cts/scheduler/rsc-defaults.xml - -diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in -index b83f812..9022ce9 100644 ---- a/cts/cts-scheduler.in -+++ b/cts/cts-scheduler.in -@@ -963,7 +963,8 @@ TESTS = [ - [ "shutdown-lock-expiration", "Ensure shutdown lock expiration works properly" ], - ], - [ -- [ "op-defaults", "Test op_defaults conditional expressions " ], -+ [ "op-defaults", "Test op_defaults conditional expressions" ], -+ [ "rsc-defaults", "Test rsc_defaults conditional expressions" ], - ], - - # @TODO: If pacemaker implements versioned attributes, uncomment these tests -diff --git a/cts/scheduler/rsc-defaults.dot b/cts/scheduler/rsc-defaults.dot -new file mode 100644 -index 0000000..d776614 ---- /dev/null -+++ b/cts/scheduler/rsc-defaults.dot -@@ -0,0 +1,18 @@ -+ digraph "g" { -+"dummy-rsc_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"dummy-rsc_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"fencing_monitor_0 cluster01" -> "fencing_start_0 cluster01" [ style = bold] -+"fencing_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"fencing_monitor_0 cluster02" -> "fencing_start_0 cluster01" [ style = bold] -+"fencing_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"fencing_start_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ip-rsc2_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ip-rsc2_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"ip-rsc_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ip-rsc_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"ping-rsc-ping_monitor_0 cluster01" -> "ping-rsc-ping_start_0 cluster02" [ style = bold] -+"ping-rsc-ping_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ping-rsc-ping_monitor_0 cluster02" -> "ping-rsc-ping_start_0 cluster02" [ style = bold] -+"ping-rsc-ping_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"ping-rsc-ping_start_0 cluster02" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/rsc-defaults.exp b/cts/scheduler/rsc-defaults.exp -new file mode 100644 -index 0000000..4aec360 ---- /dev/null -+++ b/cts/scheduler/rsc-defaults.exp -@@ -0,0 +1,124 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/rsc-defaults.scores b/cts/scheduler/rsc-defaults.scores -new file mode 100644 -index 0000000..e7f1bab ---- /dev/null -+++ b/cts/scheduler/rsc-defaults.scores -@@ -0,0 +1,11 @@ -+Allocation scores: -+pcmk__native_allocate: dummy-rsc allocation score on cluster01: 0 -+pcmk__native_allocate: dummy-rsc allocation score on cluster02: 0 -+pcmk__native_allocate: fencing allocation score on cluster01: 0 -+pcmk__native_allocate: fencing allocation score on cluster02: 0 -+pcmk__native_allocate: ip-rsc allocation score on cluster01: -INFINITY -+pcmk__native_allocate: ip-rsc allocation score on cluster02: -INFINITY -+pcmk__native_allocate: ip-rsc2 allocation score on cluster01: -INFINITY -+pcmk__native_allocate: ip-rsc2 allocation score on cluster02: -INFINITY -+pcmk__native_allocate: ping-rsc-ping allocation score on cluster01: 0 -+pcmk__native_allocate: ping-rsc-ping allocation score on cluster02: 0 -diff --git a/cts/scheduler/rsc-defaults.summary b/cts/scheduler/rsc-defaults.summary -new file mode 100644 -index 0000000..0066f2e ---- /dev/null -+++ b/cts/scheduler/rsc-defaults.summary -@@ -0,0 +1,38 @@ -+2 of 5 resource instances DISABLED and 0 BLOCKED from further action due to failure -+ -+Current cluster status: -+Online: [ cluster01 cluster02 ] -+ -+ fencing (stonith:fence_xvm): Stopped -+ ip-rsc (ocf::heartbeat:IPaddr2): Stopped (disabled) -+ ip-rsc2 (ocf::heartbeat:IPaddr2): Stopped (disabled) -+ dummy-rsc (ocf::pacemaker:Dummy): Stopped (unmanaged) -+ ping-rsc-ping (ocf::pacemaker:ping): Stopped -+ -+Transition Summary: -+ * Start fencing ( cluster01 ) -+ * Start ping-rsc-ping ( cluster02 ) -+ -+Executing cluster transition: -+ * Resource action: fencing monitor on cluster02 -+ * Resource action: fencing monitor on cluster01 -+ * Resource action: ip-rsc monitor on cluster02 -+ * Resource action: ip-rsc monitor on cluster01 -+ * Resource action: ip-rsc2 monitor on cluster02 -+ * Resource action: ip-rsc2 monitor on cluster01 -+ * Resource action: dummy-rsc monitor on cluster02 -+ * Resource action: dummy-rsc monitor on cluster01 -+ * Resource action: ping-rsc-ping monitor on cluster02 -+ * Resource action: ping-rsc-ping monitor on cluster01 -+ * Resource action: fencing start on cluster01 -+ * Resource action: ping-rsc-ping start on cluster02 -+ -+Revised cluster status: -+Online: [ cluster01 cluster02 ] -+ -+ fencing (stonith:fence_xvm): Started cluster01 -+ ip-rsc (ocf::heartbeat:IPaddr2): Stopped (disabled) -+ ip-rsc2 (ocf::heartbeat:IPaddr2): Stopped (disabled) -+ dummy-rsc (ocf::pacemaker:Dummy): Stopped (unmanaged) -+ ping-rsc-ping (ocf::pacemaker:ping): Started cluster02 -+ -diff --git a/cts/scheduler/rsc-defaults.xml b/cts/scheduler/rsc-defaults.xml -new file mode 100644 -index 0000000..38cae8b ---- /dev/null -+++ b/cts/scheduler/rsc-defaults.xml -@@ -0,0 +1,78 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - - -From bcfe068ccb3f3cb6cc3509257fbc4a59bc2b1a41 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Wed, 13 May 2020 12:47:35 -0400 -Subject: [PATCH 14/17] Test: scheduler: Add a regression test for op_defaults - with an AND expr. - ---- - cts/cts-scheduler.in | 1 + - cts/scheduler/op-defaults-2.dot | 33 ++++++ - cts/scheduler/op-defaults-2.exp | 211 ++++++++++++++++++++++++++++++++++++ - cts/scheduler/op-defaults-2.scores | 11 ++ - cts/scheduler/op-defaults-2.summary | 46 ++++++++ - cts/scheduler/op-defaults-2.xml | 73 +++++++++++++ - 6 files changed, 375 insertions(+) - create mode 100644 cts/scheduler/op-defaults-2.dot - create mode 100644 cts/scheduler/op-defaults-2.exp - create mode 100644 cts/scheduler/op-defaults-2.scores - create mode 100644 cts/scheduler/op-defaults-2.summary - create mode 100644 cts/scheduler/op-defaults-2.xml - -diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in -index 9022ce9..669b344 100644 ---- a/cts/cts-scheduler.in -+++ b/cts/cts-scheduler.in -@@ -964,6 +964,7 @@ TESTS = [ - ], - [ - [ "op-defaults", "Test op_defaults conditional expressions" ], -+ [ "op-defaults-2", "Test op_defaults AND'ed conditional expressions" ], - [ "rsc-defaults", "Test rsc_defaults conditional expressions" ], - ], - -diff --git a/cts/scheduler/op-defaults-2.dot b/cts/scheduler/op-defaults-2.dot -new file mode 100644 -index 0000000..5c67bd8 ---- /dev/null -+++ b/cts/scheduler/op-defaults-2.dot -@@ -0,0 +1,33 @@ -+ digraph "g" { -+"dummy-rsc_monitor_0 cluster01" -> "dummy-rsc_start_0 cluster02" [ style = bold] -+"dummy-rsc_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"dummy-rsc_monitor_0 cluster02" -> "dummy-rsc_start_0 cluster02" [ style = bold] -+"dummy-rsc_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"dummy-rsc_monitor_10000 cluster02" [ style=bold color="green" fontcolor="black"] -+"dummy-rsc_start_0 cluster02" -> "dummy-rsc_monitor_10000 cluster02" [ style = bold] -+"dummy-rsc_start_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"fencing_monitor_0 cluster01" -> "fencing_start_0 cluster01" [ style = bold] -+"fencing_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"fencing_monitor_0 cluster02" -> "fencing_start_0 cluster01" [ style = bold] -+"fencing_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"fencing_start_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ip-rsc_monitor_0 cluster01" -> "ip-rsc_start_0 cluster02" [ style = bold] -+"ip-rsc_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ip-rsc_monitor_0 cluster02" -> "ip-rsc_start_0 cluster02" [ style = bold] -+"ip-rsc_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"ip-rsc_monitor_20000 cluster02" [ style=bold color="green" fontcolor="black"] -+"ip-rsc_start_0 cluster02" -> "ip-rsc_monitor_20000 cluster02" [ style = bold] -+"ip-rsc_start_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"ping-rsc-ping_monitor_0 cluster01" -> "ping-rsc-ping_start_0 cluster01" [ style = bold] -+"ping-rsc-ping_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ping-rsc-ping_monitor_0 cluster02" -> "ping-rsc-ping_start_0 cluster01" [ style = bold] -+"ping-rsc-ping_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"ping-rsc-ping_start_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"rsc-passes_monitor_0 cluster01" -> "rsc-passes_start_0 cluster01" [ style = bold] -+"rsc-passes_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"rsc-passes_monitor_0 cluster02" -> "rsc-passes_start_0 cluster01" [ style = bold] -+"rsc-passes_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"rsc-passes_monitor_10000 cluster01" [ style=bold color="green" fontcolor="black"] -+"rsc-passes_start_0 cluster01" -> "rsc-passes_monitor_10000 cluster01" [ style = bold] -+"rsc-passes_start_0 cluster01" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/op-defaults-2.exp b/cts/scheduler/op-defaults-2.exp -new file mode 100644 -index 0000000..4324fde ---- /dev/null -+++ b/cts/scheduler/op-defaults-2.exp -@@ -0,0 +1,211 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/op-defaults-2.scores b/cts/scheduler/op-defaults-2.scores -new file mode 100644 -index 0000000..180c8b4 ---- /dev/null -+++ b/cts/scheduler/op-defaults-2.scores -@@ -0,0 +1,11 @@ -+Allocation scores: -+pcmk__native_allocate: dummy-rsc allocation score on cluster01: 0 -+pcmk__native_allocate: dummy-rsc allocation score on cluster02: 0 -+pcmk__native_allocate: fencing allocation score on cluster01: 0 -+pcmk__native_allocate: fencing allocation score on cluster02: 0 -+pcmk__native_allocate: ip-rsc allocation score on cluster01: 0 -+pcmk__native_allocate: ip-rsc allocation score on cluster02: 0 -+pcmk__native_allocate: ping-rsc-ping allocation score on cluster01: 0 -+pcmk__native_allocate: ping-rsc-ping allocation score on cluster02: 0 -+pcmk__native_allocate: rsc-passes allocation score on cluster01: 0 -+pcmk__native_allocate: rsc-passes allocation score on cluster02: 0 -diff --git a/cts/scheduler/op-defaults-2.summary b/cts/scheduler/op-defaults-2.summary -new file mode 100644 -index 0000000..16a68be ---- /dev/null -+++ b/cts/scheduler/op-defaults-2.summary -@@ -0,0 +1,46 @@ -+ -+Current cluster status: -+Online: [ cluster01 cluster02 ] -+ -+ fencing (stonith:fence_xvm): Stopped -+ ip-rsc (ocf::heartbeat:IPaddr2): Stopped -+ rsc-passes (ocf::heartbeat:IPaddr2): Stopped -+ dummy-rsc (ocf::pacemaker:Dummy): Stopped -+ ping-rsc-ping (ocf::pacemaker:ping): Stopped -+ -+Transition Summary: -+ * Start fencing ( cluster01 ) -+ * Start ip-rsc ( cluster02 ) -+ * Start rsc-passes ( cluster01 ) -+ * Start dummy-rsc ( cluster02 ) -+ * Start ping-rsc-ping ( cluster01 ) -+ -+Executing cluster transition: -+ * Resource action: fencing monitor on cluster02 -+ * Resource action: fencing monitor on cluster01 -+ * Resource action: ip-rsc monitor on cluster02 -+ * Resource action: ip-rsc monitor on cluster01 -+ * Resource action: rsc-passes monitor on cluster02 -+ * Resource action: rsc-passes monitor on cluster01 -+ * Resource action: dummy-rsc monitor on cluster02 -+ * Resource action: dummy-rsc monitor on cluster01 -+ * Resource action: ping-rsc-ping monitor on cluster02 -+ * Resource action: ping-rsc-ping monitor on cluster01 -+ * Resource action: fencing start on cluster01 -+ * Resource action: ip-rsc start on cluster02 -+ * Resource action: rsc-passes start on cluster01 -+ * Resource action: dummy-rsc start on cluster02 -+ * Resource action: ping-rsc-ping start on cluster01 -+ * Resource action: ip-rsc monitor=20000 on cluster02 -+ * Resource action: rsc-passes monitor=10000 on cluster01 -+ * Resource action: dummy-rsc monitor=10000 on cluster02 -+ -+Revised cluster status: -+Online: [ cluster01 cluster02 ] -+ -+ fencing (stonith:fence_xvm): Started cluster01 -+ ip-rsc (ocf::heartbeat:IPaddr2): Started cluster02 -+ rsc-passes (ocf::heartbeat:IPaddr2): Started cluster01 -+ dummy-rsc (ocf::pacemaker:Dummy): Started cluster02 -+ ping-rsc-ping (ocf::pacemaker:ping): Started cluster01 -+ -diff --git a/cts/scheduler/op-defaults-2.xml b/cts/scheduler/op-defaults-2.xml -new file mode 100644 -index 0000000..9f3c288 ---- /dev/null -+++ b/cts/scheduler/op-defaults-2.xml -@@ -0,0 +1,73 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - - -From 017b783c2037d641c40a39dd7ec3a9eba0aaa6df Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Wed, 13 May 2020 15:18:28 -0400 -Subject: [PATCH 15/17] Doc: Pacemaker Explained: Add documentation for - rsc_expr and op_expr. - ---- - doc/Pacemaker_Explained/en-US/Ch-Rules.txt | 174 +++++++++++++++++++++++++++++ - 1 file changed, 174 insertions(+) - -diff --git a/doc/Pacemaker_Explained/en-US/Ch-Rules.txt b/doc/Pacemaker_Explained/en-US/Ch-Rules.txt -index 9d617f6..5df5f82 100644 ---- a/doc/Pacemaker_Explained/en-US/Ch-Rules.txt -+++ b/doc/Pacemaker_Explained/en-US/Ch-Rules.txt -@@ -522,6 +522,124 @@ You may wish to write +end="2005-03-31T23:59:59"+ to avoid confusion. - ------- - ===== - -+== Resource Expressions == -+ -+An +rsc_expression+ is a rule condition based on a resource agent's properties. -+This rule is only valid within an +rsc_defaults+ or +op_defaults+ context. None -+of the matching attributes of +class+, +provider+, and +type+ are required. If -+one is omitted, all values of that attribute will match. For instance, omitting -++type+ means every type will match. -+ -+.Attributes of an rsc_expression Element -+[width="95%",cols="2m,<5",options="header",align="center"] -+|========================================================= -+ -+|Field -+|Description -+ -+|id -+|A unique name for the expression (required) -+ indexterm:[XML attribute,id attribute,rsc_expression element] -+ indexterm:[XML element,rsc_expression element,id attribute] -+ -+|class -+|The standard name to be matched against resource agents -+ indexterm:[XML attribute,class attribute,rsc_expression element] -+ indexterm:[XML element,rsc_expression element,class attribute] -+ -+|provider -+|If given, the vendor to be matched against resource agents. This -+ only makes sense for agents using the OCF spec. -+ indexterm:[XML attribute,provider attribute,rsc_expression element] -+ indexterm:[XML element,rsc_expression element,provider attribute] -+ -+|type -+|The name of the resource agent to be matched -+ indexterm:[XML attribute,type attribute,rsc_expression element] -+ indexterm:[XML element,rsc_expression element,type attribute] -+ -+|========================================================= -+ -+=== Example Resource-Based Expressions === -+ -+A small sample of how resource-based expressions can be used: -+ -+.True for all ocf:heartbeat:IPaddr2 resources -+==== -+[source,XML] -+---- -+ -+ -+ -+---- -+==== -+ -+.Provider doesn't apply to non-OCF resources -+==== -+[source,XML] -+---- -+ -+ -+ -+---- -+==== -+ -+== Operation Expressions == -+ -+An +op_expression+ is a rule condition based on an action of some resource -+agent. This rule is only valid within an +op_defaults+ context. -+ -+.Attributes of an op_expression Element -+[width="95%",cols="2m,<5",options="header",align="center"] -+|========================================================= -+ -+|Field -+|Description -+ -+|id -+|A unique name for the expression (required) -+ indexterm:[XML attribute,id attribute,op_expression element] -+ indexterm:[XML element,op_expression element,id attribute] -+ -+|name -+|The action name to match against. This can be any action supported by -+ the resource agent; common values include +monitor+, +start+, and +stop+ -+ (required). -+ indexterm:[XML attribute,name attribute,op_expression element] -+ indexterm:[XML element,op_expression element,name attribute] -+ -+|interval -+|The interval of the action to match against. If not given, only -+ the name attribute will be used to match. -+ indexterm:[XML attribute,interval attribute,op_expression element] -+ indexterm:[XML element,op_expression element,interval attribute] -+ -+|========================================================= -+ -+=== Example Operation-Based Expressions === -+ -+A small sample of how operation-based expressions can be used: -+ -+.True for all monitor actions -+==== -+[source,XML] -+---- -+ -+ -+ -+---- -+==== -+ -+.True for all monitor actions with a 10 second interval -+==== -+[source,XML] -+---- -+ -+ -+ -+---- -+==== -+ - == Using Rules to Determine Resource Location == - indexterm:[Rule,Determine Resource Location] - indexterm:[Resource,Location,Determine by Rules] -@@ -710,6 +828,62 @@ Rules may be used similarly in +instance_attributes+ or +utilization+ blocks. - Any single block may directly contain only a single rule, but that rule may - itself contain any number of rules. - -++rsc_expression+ and +op_expression+ blocks may additionally be used to set defaults -+on either a single resource or across an entire class of resources with a single -+rule. +rsc_expression+ may be used to select resource agents within both +rsc_defaults+ -+and +op_defaults+, while +op_expression+ may only be used within +op_defaults+. If -+multiple rules succeed for a given resource agent, the last one specified will be -+the one that takes effect. As with any other rule, boolean operations may be used -+to make more complicated expressions. -+ -+.Set all IPaddr2 resources to stopped -+===== -+[source,XML] -+------- -+ -+ -+ -+ -+ -+ -+ -+ -+------- -+===== -+ -+.Set all monitor action timeouts to 7 seconds -+===== -+[source,XML] -+------- -+ -+ -+ -+ -+ -+ -+ -+ -+------- -+===== -+ -+.Set the monitor action timeout on all IPaddr2 resources with a given monitor interval to 8 seconds -+===== -+[source,XML] -+------- -+ -+ -+ -+ -+ -+ -+ -+ -+ -+------- -+===== -+ - === Using Rules to Control Cluster Options === - indexterm:[Rule,Controlling Cluster Options] - indexterm:[Cluster,Setting Options with Rules] --- -1.8.3.1 - - -From b8dd16c5e454445f73416ae8b74649545ee1b472 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Wed, 13 May 2020 16:26:21 -0400 -Subject: [PATCH 16/17] Test: scheduler: Add a test for multiple rules applying - to the same resource. - ---- - cts/cts-scheduler.in | 1 + - cts/scheduler/op-defaults-3.dot | 14 +++++++ - cts/scheduler/op-defaults-3.exp | 83 +++++++++++++++++++++++++++++++++++++ - cts/scheduler/op-defaults-3.scores | 5 +++ - cts/scheduler/op-defaults-3.summary | 26 ++++++++++++ - cts/scheduler/op-defaults-3.xml | 54 ++++++++++++++++++++++++ - 6 files changed, 183 insertions(+) - create mode 100644 cts/scheduler/op-defaults-3.dot - create mode 100644 cts/scheduler/op-defaults-3.exp - create mode 100644 cts/scheduler/op-defaults-3.scores - create mode 100644 cts/scheduler/op-defaults-3.summary - create mode 100644 cts/scheduler/op-defaults-3.xml - -diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in -index 669b344..2c2d14f 100644 ---- a/cts/cts-scheduler.in -+++ b/cts/cts-scheduler.in -@@ -965,6 +965,7 @@ TESTS = [ - [ - [ "op-defaults", "Test op_defaults conditional expressions" ], - [ "op-defaults-2", "Test op_defaults AND'ed conditional expressions" ], -+ [ "op-defaults-3", "Test op_defaults precedence" ], - [ "rsc-defaults", "Test rsc_defaults conditional expressions" ], - ], - -diff --git a/cts/scheduler/op-defaults-3.dot b/cts/scheduler/op-defaults-3.dot -new file mode 100644 -index 0000000..382f630 ---- /dev/null -+++ b/cts/scheduler/op-defaults-3.dot -@@ -0,0 +1,14 @@ -+ digraph "g" { -+"dummy-rsc_monitor_0 cluster01" -> "dummy-rsc_start_0 cluster02" [ style = bold] -+"dummy-rsc_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"dummy-rsc_monitor_0 cluster02" -> "dummy-rsc_start_0 cluster02" [ style = bold] -+"dummy-rsc_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"dummy-rsc_monitor_10000 cluster02" [ style=bold color="green" fontcolor="black"] -+"dummy-rsc_start_0 cluster02" -> "dummy-rsc_monitor_10000 cluster02" [ style = bold] -+"dummy-rsc_start_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"fencing_monitor_0 cluster01" -> "fencing_start_0 cluster01" [ style = bold] -+"fencing_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"fencing_monitor_0 cluster02" -> "fencing_start_0 cluster01" [ style = bold] -+"fencing_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"fencing_start_0 cluster01" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/op-defaults-3.exp b/cts/scheduler/op-defaults-3.exp -new file mode 100644 -index 0000000..6d567dc ---- /dev/null -+++ b/cts/scheduler/op-defaults-3.exp -@@ -0,0 +1,83 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/op-defaults-3.scores b/cts/scheduler/op-defaults-3.scores -new file mode 100644 -index 0000000..0a5190a ---- /dev/null -+++ b/cts/scheduler/op-defaults-3.scores -@@ -0,0 +1,5 @@ -+Allocation scores: -+pcmk__native_allocate: dummy-rsc allocation score on cluster01: 0 -+pcmk__native_allocate: dummy-rsc allocation score on cluster02: 0 -+pcmk__native_allocate: fencing allocation score on cluster01: 0 -+pcmk__native_allocate: fencing allocation score on cluster02: 0 -diff --git a/cts/scheduler/op-defaults-3.summary b/cts/scheduler/op-defaults-3.summary -new file mode 100644 -index 0000000..a83eb15 ---- /dev/null -+++ b/cts/scheduler/op-defaults-3.summary -@@ -0,0 +1,26 @@ -+ -+Current cluster status: -+Online: [ cluster01 cluster02 ] -+ -+ fencing (stonith:fence_xvm): Stopped -+ dummy-rsc (ocf::pacemaker:Dummy): Stopped -+ -+Transition Summary: -+ * Start fencing ( cluster01 ) -+ * Start dummy-rsc ( cluster02 ) -+ -+Executing cluster transition: -+ * Resource action: fencing monitor on cluster02 -+ * Resource action: fencing monitor on cluster01 -+ * Resource action: dummy-rsc monitor on cluster02 -+ * Resource action: dummy-rsc monitor on cluster01 -+ * Resource action: fencing start on cluster01 -+ * Resource action: dummy-rsc start on cluster02 -+ * Resource action: dummy-rsc monitor=10000 on cluster02 -+ -+Revised cluster status: -+Online: [ cluster01 cluster02 ] -+ -+ fencing (stonith:fence_xvm): Started cluster01 -+ dummy-rsc (ocf::pacemaker:Dummy): Started cluster02 -+ -diff --git a/cts/scheduler/op-defaults-3.xml b/cts/scheduler/op-defaults-3.xml -new file mode 100644 -index 0000000..4a8912e ---- /dev/null -+++ b/cts/scheduler/op-defaults-3.xml -@@ -0,0 +1,54 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - - -From b9ccde16609e7d005ac0578a603da97a1808704a Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Fri, 15 May 2020 13:48:47 -0400 -Subject: [PATCH 17/17] Test: scheduler: Add a test for rsc_defaults not - specifying type. - ---- - cts/cts-scheduler.in | 1 + - cts/scheduler/rsc-defaults-2.dot | 11 ++++++ - cts/scheduler/rsc-defaults-2.exp | 72 ++++++++++++++++++++++++++++++++++++ - cts/scheduler/rsc-defaults-2.scores | 7 ++++ - cts/scheduler/rsc-defaults-2.summary | 27 ++++++++++++++ - cts/scheduler/rsc-defaults-2.xml | 52 ++++++++++++++++++++++++++ - 6 files changed, 170 insertions(+) - create mode 100644 cts/scheduler/rsc-defaults-2.dot - create mode 100644 cts/scheduler/rsc-defaults-2.exp - create mode 100644 cts/scheduler/rsc-defaults-2.scores - create mode 100644 cts/scheduler/rsc-defaults-2.summary - create mode 100644 cts/scheduler/rsc-defaults-2.xml - -diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in -index 2c2d14f..346ada2 100644 ---- a/cts/cts-scheduler.in -+++ b/cts/cts-scheduler.in -@@ -967,6 +967,7 @@ TESTS = [ - [ "op-defaults-2", "Test op_defaults AND'ed conditional expressions" ], - [ "op-defaults-3", "Test op_defaults precedence" ], - [ "rsc-defaults", "Test rsc_defaults conditional expressions" ], -+ [ "rsc-defaults-2", "Test rsc_defaults conditional expressions without type" ], - ], - - # @TODO: If pacemaker implements versioned attributes, uncomment these tests -diff --git a/cts/scheduler/rsc-defaults-2.dot b/cts/scheduler/rsc-defaults-2.dot -new file mode 100644 -index 0000000..b43c5e6 ---- /dev/null -+++ b/cts/scheduler/rsc-defaults-2.dot -@@ -0,0 +1,11 @@ -+ digraph "g" { -+"dummy-rsc_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"dummy-rsc_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"fencing_monitor_0 cluster01" -> "fencing_start_0 cluster01" [ style = bold] -+"fencing_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"fencing_monitor_0 cluster02" -> "fencing_start_0 cluster01" [ style = bold] -+"fencing_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+"fencing_start_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ping-rsc-ping_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"] -+"ping-rsc-ping_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/rsc-defaults-2.exp b/cts/scheduler/rsc-defaults-2.exp -new file mode 100644 -index 0000000..e9e1b5f ---- /dev/null -+++ b/cts/scheduler/rsc-defaults-2.exp -@@ -0,0 +1,72 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/rsc-defaults-2.scores b/cts/scheduler/rsc-defaults-2.scores -new file mode 100644 -index 0000000..4b70f54 ---- /dev/null -+++ b/cts/scheduler/rsc-defaults-2.scores -@@ -0,0 +1,7 @@ -+Allocation scores: -+pcmk__native_allocate: dummy-rsc allocation score on cluster01: 0 -+pcmk__native_allocate: dummy-rsc allocation score on cluster02: 0 -+pcmk__native_allocate: fencing allocation score on cluster01: 0 -+pcmk__native_allocate: fencing allocation score on cluster02: 0 -+pcmk__native_allocate: ping-rsc-ping allocation score on cluster01: 0 -+pcmk__native_allocate: ping-rsc-ping allocation score on cluster02: 0 -diff --git a/cts/scheduler/rsc-defaults-2.summary b/cts/scheduler/rsc-defaults-2.summary -new file mode 100644 -index 0000000..46a2a2d ---- /dev/null -+++ b/cts/scheduler/rsc-defaults-2.summary -@@ -0,0 +1,27 @@ -+ -+Current cluster status: -+Online: [ cluster01 cluster02 ] -+ -+ fencing (stonith:fence_xvm): Stopped -+ dummy-rsc (ocf::pacemaker:Dummy): Stopped (unmanaged) -+ ping-rsc-ping (ocf::pacemaker:ping): Stopped (unmanaged) -+ -+Transition Summary: -+ * Start fencing ( cluster01 ) -+ -+Executing cluster transition: -+ * Resource action: fencing monitor on cluster02 -+ * Resource action: fencing monitor on cluster01 -+ * Resource action: dummy-rsc monitor on cluster02 -+ * Resource action: dummy-rsc monitor on cluster01 -+ * Resource action: ping-rsc-ping monitor on cluster02 -+ * Resource action: ping-rsc-ping monitor on cluster01 -+ * Resource action: fencing start on cluster01 -+ -+Revised cluster status: -+Online: [ cluster01 cluster02 ] -+ -+ fencing (stonith:fence_xvm): Started cluster01 -+ dummy-rsc (ocf::pacemaker:Dummy): Stopped (unmanaged) -+ ping-rsc-ping (ocf::pacemaker:ping): Stopped (unmanaged) -+ -diff --git a/cts/scheduler/rsc-defaults-2.xml b/cts/scheduler/rsc-defaults-2.xml -new file mode 100644 -index 0000000..a160fae ---- /dev/null -+++ b/cts/scheduler/rsc-defaults-2.xml -@@ -0,0 +1,52 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - diff --git a/SOURCES/002-demote.patch b/SOURCES/002-demote.patch deleted file mode 100644 index 5da2515..0000000 --- a/SOURCES/002-demote.patch +++ /dev/null @@ -1,8664 +0,0 @@ -From f1f71b3f3c342987db0058e7db0030417f3f83fa Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 28 May 2020 08:22:00 -0500 -Subject: [PATCH 01/20] Refactor: scheduler: functionize comparing on-fail - values - -The action_fail_response enum values used for the "on-fail" operation -meta-attribute were initially intended to be in order of severity. -However as new values were added, they were added to the end (out of severity -order) to preserve API backward compatibility. - -This resulted in a convoluted comparison of values that will only get worse as -more values are added. - -This commit adds a comparison function to isolate that complexity. ---- - include/crm/pengine/common.h | 32 ++++++++++++------ - lib/pengine/unpack.c | 80 +++++++++++++++++++++++++++++++++++++++++--- - 2 files changed, 97 insertions(+), 15 deletions(-) - -diff --git a/include/crm/pengine/common.h b/include/crm/pengine/common.h -index 3a770b7..2737b2e 100644 ---- a/include/crm/pengine/common.h -+++ b/include/crm/pengine/common.h -@@ -22,18 +22,29 @@ extern "C" { - extern gboolean was_processing_error; - extern gboolean was_processing_warning; - --/* order is significant here -- * items listed in order of accending severeness -- * more severe actions take precedent over lower ones -+/* The order is (partially) significant here; the values from action_fail_ignore -+ * through action_fail_fence are in order of increasing severity. -+ * -+ * @COMPAT The values should be ordered and numbered per the "TODO" comments -+ * below, so all values are in order of severity and there is room for -+ * future additions, but that would break API compatibility. -+ * @TODO For now, we just use a function to compare the values specially, but -+ * at the next compatibility break, we should arrange things properly. - */ - enum action_fail_response { -- action_fail_ignore, -- action_fail_recover, -- action_fail_migrate, /* recover by moving it somewhere else */ -- action_fail_block, -- action_fail_stop, -- action_fail_standby, -- action_fail_fence, -+ action_fail_ignore, // @TODO = 10 -+ // @TODO action_fail_demote = 20, -+ action_fail_recover, // @TODO = 30 -+ // @TODO action_fail_reset_remote = 40, -+ // @TODO action_fail_restart_container = 50, -+ action_fail_migrate, // @TODO = 60 -+ action_fail_block, // @TODO = 70 -+ action_fail_stop, // @TODO = 80 -+ action_fail_standby, // @TODO = 90 -+ action_fail_fence, // @TODO = 100 -+ -+ // @COMPAT Values below here are out of order for API compatibility -+ - action_fail_restart_container, - - /* This is reserved for internal use for remote node connection resources. -@@ -44,6 +55,7 @@ enum action_fail_response { - */ - action_fail_reset_remote, - -+ action_fail_demote, - }; - - /* the "done" action must be the "pre" action +1 */ -diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c -index 3c6606b..f688881 100644 ---- a/lib/pengine/unpack.c -+++ b/lib/pengine/unpack.c -@@ -2770,6 +2770,78 @@ last_change_str(xmlNode *xml_op) - return ((when_s && *when_s)? when_s : "unknown time"); - } - -+/*! -+ * \internal -+ * \brief Compare two on-fail values -+ * -+ * \param[in] first One on-fail value to compare -+ * \param[in] second The other on-fail value to compare -+ * -+ * \return A negative number if second is more severe than first, zero if they -+ * are equal, or a positive number if first is more severe than second. -+ * \note This is only needed until the action_fail_response values can be -+ * renumbered at the next API compatibility break. -+ */ -+static int -+cmp_on_fail(enum action_fail_response first, enum action_fail_response second) -+{ -+ switch (first) { -+ case action_fail_reset_remote: -+ switch (second) { -+ case action_fail_ignore: -+ case action_fail_recover: -+ return 1; -+ case action_fail_reset_remote: -+ return 0; -+ default: -+ return -1; -+ } -+ break; -+ -+ case action_fail_restart_container: -+ switch (second) { -+ case action_fail_ignore: -+ case action_fail_recover: -+ case action_fail_reset_remote: -+ return 1; -+ case action_fail_restart_container: -+ return 0; -+ default: -+ return -1; -+ } -+ break; -+ -+ default: -+ break; -+ } -+ switch (second) { -+ case action_fail_reset_remote: -+ switch (first) { -+ case action_fail_ignore: -+ case action_fail_recover: -+ return -1; -+ default: -+ return 1; -+ } -+ break; -+ -+ case action_fail_restart_container: -+ switch (first) { -+ case action_fail_ignore: -+ case action_fail_recover: -+ case action_fail_reset_remote: -+ return -1; -+ default: -+ return 1; -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return first - second; -+} -+ - static void - unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * xml_op, xmlNode ** last_failure, - enum action_fail_response * on_fail, pe_working_set_t * data_set) -@@ -2829,10 +2901,7 @@ unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * x - } - - action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set); -- if ((action->on_fail <= action_fail_fence && *on_fail < action->on_fail) || -- (action->on_fail == action_fail_reset_remote && *on_fail <= action_fail_recover) || -- (action->on_fail == action_fail_restart_container && *on_fail <= action_fail_recover) || -- (*on_fail == action_fail_restart_container && action->on_fail >= action_fail_migrate)) { -+ if (cmp_on_fail(*on_fail, action->on_fail) < 0) { - pe_rsc_trace(rsc, "on-fail %s -> %s for %s (%s)", fail2text(*on_fail), - fail2text(action->on_fail), action->uuid, key); - *on_fail = action->on_fail; -@@ -3675,7 +3744,8 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op, - - record_failed_op(xml_op, node, rsc, data_set); - -- if (failure_strategy == action_fail_restart_container && *on_fail <= action_fail_recover) { -+ if ((failure_strategy == action_fail_restart_container) -+ && cmp_on_fail(*on_fail, action_fail_recover) <= 0) { - *on_fail = failure_strategy; - } - --- -1.8.3.1 - - -From ef246ff05d7459f9672b10ac1873e3191a3b46e9 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 28 May 2020 08:27:47 -0500 -Subject: [PATCH 02/20] Fix: scheduler: disallow on-fail=stop for stop - operations - -because it would loop infinitely as long as the stop continued to fail ---- - lib/pengine/utils.c | 15 ++++++++++++--- - 1 file changed, 12 insertions(+), 3 deletions(-) - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index 20a8db5..3fb7e62 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -716,16 +716,25 @@ custom_action(pe_resource_t * rsc, char *key, const char *task, - return action; - } - -+static bool -+valid_stop_on_fail(const char *value) -+{ -+ return safe_str_neq(value, "standby") -+ && safe_str_neq(value, "stop"); -+} -+ - static const char * - unpack_operation_on_fail(pe_action_t * action) - { - - const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL); - -- if (safe_str_eq(action->task, CRMD_ACTION_STOP) && safe_str_eq(value, "standby")) { -+ if (safe_str_eq(action->task, CRMD_ACTION_STOP) -+ && !valid_stop_on_fail(value)) { -+ - pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop " -- "action to default value because 'standby' is not " -- "allowed for stop", action->rsc->id); -+ "action to default value because '%s' is not " -+ "allowed for stop", action->rsc->id, value); - return NULL; - } else if (safe_str_eq(action->task, CRMD_ACTION_DEMOTE) && !value) { - /* demote on_fail defaults to master monitor value if present */ --- -1.8.3.1 - - -From 8dceba792ffe65cd77c3aae430067638dbba63f9 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 28 May 2020 08:50:33 -0500 -Subject: [PATCH 03/20] Refactor: scheduler: use more appropriate types in a - couple places - ---- - lib/pengine/unpack.c | 5 ++--- - 1 file changed, 2 insertions(+), 3 deletions(-) - -diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c -index f688881..6a350e5 100644 ---- a/lib/pengine/unpack.c -+++ b/lib/pengine/unpack.c -@@ -2244,7 +2244,7 @@ unpack_lrm_rsc_state(pe_node_t * node, xmlNode * rsc_entry, pe_working_set_t * d - xmlNode *rsc_op = NULL; - xmlNode *last_failure = NULL; - -- enum action_fail_response on_fail = FALSE; -+ enum action_fail_response on_fail = action_fail_ignore; - enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN; - - crm_trace("[%s] Processing %s on %s", -@@ -2287,7 +2287,6 @@ unpack_lrm_rsc_state(pe_node_t * node, xmlNode * rsc_entry, pe_working_set_t * d - - /* process operations */ - saved_role = rsc->role; -- on_fail = action_fail_ignore; - rsc->role = RSC_ROLE_UNKNOWN; - sorted_op_list = g_list_sort(op_list, sort_op_by_callid); - -@@ -3376,7 +3375,7 @@ int pe__target_rc_from_xml(xmlNode *xml_op) - static enum action_fail_response - get_action_on_fail(pe_resource_t *rsc, const char *key, const char *task, pe_working_set_t * data_set) - { -- int result = action_fail_recover; -+ enum action_fail_response result = action_fail_recover; - pe_action_t *action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set); - - result = action->on_fail; --- -1.8.3.1 - - -From a4d6a20a990d1461184f888e21aa61cddff8996d Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 2 Jun 2020 12:05:57 -0500 -Subject: [PATCH 04/20] Low: libpacemaker: don't force stop when skipping - reload of failed resource - -Normal failure recovery will apply, which will stop if needed. - -(The stop was forced as of 2558d76f.) ---- - lib/pacemaker/pcmk_sched_native.c | 16 +++++++++++++--- - 1 file changed, 13 insertions(+), 3 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c -index bd8a0b5..ff2fb92 100644 ---- a/lib/pacemaker/pcmk_sched_native.c -+++ b/lib/pacemaker/pcmk_sched_native.c -@@ -3362,9 +3362,19 @@ ReloadRsc(pe_resource_t * rsc, pe_node_t *node, pe_working_set_t * data_set) - pe_rsc_trace(rsc, "%s: unmanaged", rsc->id); - return; - -- } else if (is_set(rsc->flags, pe_rsc_failed) || is_set(rsc->flags, pe_rsc_start_pending)) { -- pe_rsc_trace(rsc, "%s: general resource state: flags=0x%.16llx", rsc->id, rsc->flags); -- stop_action(rsc, node, FALSE); /* Force a full restart, overkill? */ -+ } else if (is_set(rsc->flags, pe_rsc_failed)) { -+ /* We don't need to specify any particular actions here, normal failure -+ * recovery will apply. -+ */ -+ pe_rsc_trace(rsc, "%s: preventing reload because failed", rsc->id); -+ return; -+ -+ } else if (is_set(rsc->flags, pe_rsc_start_pending)) { -+ /* If a resource's configuration changed while a start was pending, -+ * force a full restart. -+ */ -+ pe_rsc_trace(rsc, "%s: preventing reload because start pending", rsc->id); -+ stop_action(rsc, node, FALSE); - return; - - } else if (node == NULL) { --- -1.8.3.1 - - -From f2d244bc4306297d5960c0ba54e0a85a68e864ee Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 2 Jun 2020 12:16:33 -0500 -Subject: [PATCH 05/20] Test: scheduler: test forcing a restart instead of - reload when start is pending - ---- - cts/cts-scheduler.in | 1 + - cts/scheduler/params-3.dot | 28 ++++++ - cts/scheduler/params-3.exp | 208 +++++++++++++++++++++++++++++++++++++++++ - cts/scheduler/params-3.scores | 21 +++++ - cts/scheduler/params-3.summary | 45 +++++++++ - cts/scheduler/params-3.xml | 154 ++++++++++++++++++++++++++++++ - 6 files changed, 457 insertions(+) - create mode 100644 cts/scheduler/params-3.dot - create mode 100644 cts/scheduler/params-3.exp - create mode 100644 cts/scheduler/params-3.scores - create mode 100644 cts/scheduler/params-3.summary - create mode 100644 cts/scheduler/params-3.xml - -diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in -index 346ada2..ae8247e 100644 ---- a/cts/cts-scheduler.in -+++ b/cts/cts-scheduler.in -@@ -84,6 +84,7 @@ TESTS = [ - [ "params-0", "Params: No change" ], - [ "params-1", "Params: Changed" ], - [ "params-2", "Params: Resource definition" ], -+ [ "params-3", "Params: Restart instead of reload if start pending" ], - [ "params-4", "Params: Reload" ], - [ "params-5", "Params: Restart based on probe digest" ], - [ "novell-251689", "Resource definition change + target_role=stopped" ], -diff --git a/cts/scheduler/params-3.dot b/cts/scheduler/params-3.dot -new file mode 100644 -index 0000000..d681ee5 ---- /dev/null -+++ b/cts/scheduler/params-3.dot -@@ -0,0 +1,28 @@ -+ digraph "g" { -+"Cancel rsc_c001n02_monitor_5000 c001n02" [ style=bold color="green" fontcolor="black"] -+"DcIPaddr_monitor_0 c001n01" -> "DcIPaddr_start_0 c001n02" [ style = bold] -+"DcIPaddr_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"] -+"DcIPaddr_monitor_0 c001n03" -> "DcIPaddr_start_0 c001n02" [ style = bold] -+"DcIPaddr_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"] -+"DcIPaddr_monitor_0 c001n08" -> "DcIPaddr_start_0 c001n02" [ style = bold] -+"DcIPaddr_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"] -+"DcIPaddr_monitor_5000 c001n02" [ style=bold color="green" fontcolor="black"] -+"DcIPaddr_start_0 c001n02" -> "DcIPaddr_monitor_5000 c001n02" [ style = bold] -+"DcIPaddr_start_0 c001n02" [ style=bold color="green" fontcolor="black"] -+"DcIPaddr_stop_0 c001n02" -> "DcIPaddr_start_0 c001n02" [ style = bold] -+"DcIPaddr_stop_0 c001n02" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n01_monitor_0 c001n02" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n01_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n01_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n02_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n02_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n02_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n02_monitor_6000 c001n02" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n03_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n03_monitor_0 c001n02" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n03_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n08_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n08_monitor_0 c001n02" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n08_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"] -+"rsc_c001n08_monitor_5000 c001n08" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/params-3.exp b/cts/scheduler/params-3.exp -new file mode 100644 -index 0000000..5cccdec ---- /dev/null -+++ b/cts/scheduler/params-3.exp -@@ -0,0 +1,208 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/params-3.scores b/cts/scheduler/params-3.scores -new file mode 100644 -index 0000000..00417ea ---- /dev/null -+++ b/cts/scheduler/params-3.scores -@@ -0,0 +1,21 @@ -+Allocation scores: -+pcmk__native_allocate: DcIPaddr allocation score on c001n01: -INFINITY -+pcmk__native_allocate: DcIPaddr allocation score on c001n02: 0 -+pcmk__native_allocate: DcIPaddr allocation score on c001n03: -INFINITY -+pcmk__native_allocate: DcIPaddr allocation score on c001n08: -INFINITY -+pcmk__native_allocate: rsc_c001n01 allocation score on c001n01: 100 -+pcmk__native_allocate: rsc_c001n01 allocation score on c001n02: 0 -+pcmk__native_allocate: rsc_c001n01 allocation score on c001n03: 0 -+pcmk__native_allocate: rsc_c001n01 allocation score on c001n08: 0 -+pcmk__native_allocate: rsc_c001n02 allocation score on c001n01: 0 -+pcmk__native_allocate: rsc_c001n02 allocation score on c001n02: 100 -+pcmk__native_allocate: rsc_c001n02 allocation score on c001n03: 0 -+pcmk__native_allocate: rsc_c001n02 allocation score on c001n08: 0 -+pcmk__native_allocate: rsc_c001n03 allocation score on c001n01: 0 -+pcmk__native_allocate: rsc_c001n03 allocation score on c001n02: 0 -+pcmk__native_allocate: rsc_c001n03 allocation score on c001n03: 100 -+pcmk__native_allocate: rsc_c001n03 allocation score on c001n08: 0 -+pcmk__native_allocate: rsc_c001n08 allocation score on c001n01: 0 -+pcmk__native_allocate: rsc_c001n08 allocation score on c001n02: 0 -+pcmk__native_allocate: rsc_c001n08 allocation score on c001n03: 0 -+pcmk__native_allocate: rsc_c001n08 allocation score on c001n08: 100 -diff --git a/cts/scheduler/params-3.summary b/cts/scheduler/params-3.summary -new file mode 100644 -index 0000000..257f8ba ---- /dev/null -+++ b/cts/scheduler/params-3.summary -@@ -0,0 +1,45 @@ -+ -+Current cluster status: -+Online: [ c001n01 c001n02 c001n03 c001n08 ] -+ -+ DcIPaddr (ocf::heartbeat:IPaddr): Starting c001n02 -+ rsc_c001n08 (ocf::heartbeat:IPaddr): Started c001n08 -+ rsc_c001n02 (ocf::heartbeat:IPaddr): Started c001n02 -+ rsc_c001n03 (ocf::heartbeat:IPaddr): Started c001n03 -+ rsc_c001n01 (ocf::heartbeat:IPaddr): Started c001n01 -+ -+Transition Summary: -+ * Restart DcIPaddr ( c001n02 ) -+ -+Executing cluster transition: -+ * Resource action: DcIPaddr monitor on c001n08 -+ * Resource action: DcIPaddr monitor on c001n03 -+ * Resource action: DcIPaddr monitor on c001n01 -+ * Resource action: DcIPaddr stop on c001n02 -+ * Resource action: rsc_c001n08 monitor on c001n03 -+ * Resource action: rsc_c001n08 monitor on c001n02 -+ * Resource action: rsc_c001n08 monitor on c001n01 -+ * Resource action: rsc_c001n08 monitor=5000 on c001n08 -+ * Resource action: rsc_c001n02 monitor=6000 on c001n02 -+ * Resource action: rsc_c001n02 monitor on c001n08 -+ * Resource action: rsc_c001n02 monitor on c001n03 -+ * Resource action: rsc_c001n02 monitor on c001n01 -+ * Resource action: rsc_c001n02 cancel=5000 on c001n02 -+ * Resource action: rsc_c001n03 monitor on c001n08 -+ * Resource action: rsc_c001n03 monitor on c001n02 -+ * Resource action: rsc_c001n03 monitor on c001n01 -+ * Resource action: rsc_c001n01 monitor on c001n08 -+ * Resource action: rsc_c001n01 monitor on c001n03 -+ * Resource action: rsc_c001n01 monitor on c001n02 -+ * Resource action: DcIPaddr start on c001n02 -+ * Resource action: DcIPaddr monitor=5000 on c001n02 -+ -+Revised cluster status: -+Online: [ c001n01 c001n02 c001n03 c001n08 ] -+ -+ DcIPaddr (ocf::heartbeat:IPaddr): Started c001n02 -+ rsc_c001n08 (ocf::heartbeat:IPaddr): Started c001n08 -+ rsc_c001n02 (ocf::heartbeat:IPaddr): Started c001n02 -+ rsc_c001n03 (ocf::heartbeat:IPaddr): Started c001n03 -+ rsc_c001n01 (ocf::heartbeat:IPaddr): Started c001n01 -+ -diff --git a/cts/scheduler/params-3.xml b/cts/scheduler/params-3.xml -new file mode 100644 -index 0000000..ee6e157 ---- /dev/null -+++ b/cts/scheduler/params-3.xml -@@ -0,0 +1,154 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - - -From ff6aebecf8b40b882bddbd0d78e3f8702f97147e Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 13 Apr 2020 12:22:35 -0500 -Subject: [PATCH 06/20] Doc: libpacemaker: improve comments when logging - actions - -... with slight refactoring for consistency ---- - lib/pacemaker/pcmk_sched_native.c | 41 ++++++++++++++++++++++----------------- - 1 file changed, 23 insertions(+), 18 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c -index ff2fb92..f14e690 100644 ---- a/lib/pacemaker/pcmk_sched_native.c -+++ b/lib/pacemaker/pcmk_sched_native.c -@@ -2348,8 +2348,6 @@ native_expand(pe_resource_t * rsc, pe_working_set_t * data_set) - } \ - } while(0) - --static int rsc_width = 5; --static int detail_width = 5; - static void - LogAction(const char *change, pe_resource_t *rsc, pe_node_t *origin, pe_node_t *destination, pe_action_t *action, pe_action_t *source, gboolean terminal) - { -@@ -2360,6 +2358,9 @@ LogAction(const char *change, pe_resource_t *rsc, pe_node_t *origin, pe_node_t * - bool same_role = FALSE; - bool need_role = FALSE; - -+ static int rsc_width = 5; -+ static int detail_width = 5; -+ - CRM_ASSERT(action); - CRM_ASSERT(destination != NULL || origin != NULL); - -@@ -2384,36 +2385,40 @@ LogAction(const char *change, pe_resource_t *rsc, pe_node_t *origin, pe_node_t * - same_role = TRUE; - } - -- if(need_role && origin == NULL) { -- /* Promoting from Stopped */ -+ if (need_role && (origin == NULL)) { -+ /* Starting and promoting a promotable clone instance */ - details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), destination->details->uname); - -- } else if(need_role && destination == NULL) { -- /* Demoting a Master or Stopping a Slave */ -+ } else if (origin == NULL) { -+ /* Starting a resource */ -+ details = crm_strdup_printf("%s", destination->details->uname); -+ -+ } else if (need_role && (destination == NULL)) { -+ /* Stopping a promotable clone instance */ - details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname); - -- } else if(origin == NULL || destination == NULL) { -- /* Starting or stopping a resource */ -- details = crm_strdup_printf("%s", origin?origin->details->uname:destination->details->uname); -+ } else if (destination == NULL) { -+ /* Stopping a resource */ -+ details = crm_strdup_printf("%s", origin->details->uname); - -- } else if(need_role && same_role && same_host) { -- /* Recovering or restarting a promotable clone resource */ -+ } else if (need_role && same_role && same_host) { -+ /* Recovering, restarting or re-promoting a promotable clone instance */ - details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname); - -- } else if(same_role && same_host) { -+ } else if (same_role && same_host) { - /* Recovering or Restarting a normal resource */ - details = crm_strdup_printf("%s", origin->details->uname); - -- } else if(same_role && need_role) { -- /* Moving a promotable clone resource */ -+ } else if (need_role && same_role) { -+ /* Moving a promotable clone instance */ - details = crm_strdup_printf("%s -> %s %s", origin->details->uname, destination->details->uname, role2text(rsc->role)); - -- } else if(same_role) { -+ } else if (same_role) { - /* Moving a normal resource */ - details = crm_strdup_printf("%s -> %s", origin->details->uname, destination->details->uname); - -- } else if(same_host) { -- /* Promoting or demoting a promotable clone resource */ -+ } else if (same_host) { -+ /* Promoting or demoting a promotable clone instance */ - details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), origin->details->uname); - - } else { -@@ -2560,7 +2565,7 @@ LogActions(pe_resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) - pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id, role2text(rsc->role), - next->details->uname); - -- } else if (start && is_set(start->flags, pe_action_runnable) == FALSE) { -+ } else if (is_not_set(start->flags, pe_action_runnable)) { - LogAction("Stop", rsc, current, NULL, stop, - (stop && stop->reason)? stop : start, terminal); - STOP_SANITY_ASSERT(__LINE__); --- -1.8.3.1 - - -From 98c3b649fa065b7e7a59029cc2f887bc462d170a Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 13 Apr 2020 12:23:22 -0500 -Subject: [PATCH 07/20] Log: libpacemaker: check for re-promotes specifically - -If a promotable clone instance is being demoted and promoted on its current -node, without also stopping and starting, it previously would be logged as -"Leave" indicating unchanged, because the current and next role are the same. - -Now, check for this situation specifically, and log it as "Re-promote". - -Currently, the scheduler is not capable of generating this situation, but -upcoming changes will. ---- - lib/pacemaker/pcmk_sched_native.c | 12 ++++++++++-- - 1 file changed, 10 insertions(+), 2 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c -index f14e690..89952bf 100644 ---- a/lib/pacemaker/pcmk_sched_native.c -+++ b/lib/pacemaker/pcmk_sched_native.c -@@ -2561,9 +2561,17 @@ LogActions(pe_resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) - } else if (is_set(rsc->flags, pe_rsc_reload)) { - LogAction("Reload", rsc, current, next, start, NULL, terminal); - -+ - } else if (start == NULL || is_set(start->flags, pe_action_optional)) { -- pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id, role2text(rsc->role), -- next->details->uname); -+ if ((demote != NULL) && (promote != NULL) -+ && is_not_set(demote->flags, pe_action_optional) -+ && is_not_set(promote->flags, pe_action_optional)) { -+ LogAction("Re-promote", rsc, current, next, promote, demote, -+ terminal); -+ } else { -+ pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id, -+ role2text(rsc->role), next->details->uname); -+ } - - } else if (is_not_set(start->flags, pe_action_runnable)) { - LogAction("Stop", rsc, current, NULL, stop, --- -1.8.3.1 - - -From fd55a6660574c0bca517fd519377340712fb443a Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 13 Apr 2020 12:51:03 -0500 -Subject: [PATCH 08/20] Doc: libpacemaker: improve comments for resource state - and action matrices - -Also, make them static, for linker efficiency. ---- - lib/pacemaker/pcmk_sched_native.c | 39 ++++++++++++++++++++++++--------------- - 1 file changed, 24 insertions(+), 15 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c -index 89952bf..b9bca80 100644 ---- a/lib/pacemaker/pcmk_sched_native.c -+++ b/lib/pacemaker/pcmk_sched_native.c -@@ -41,27 +41,36 @@ gboolean PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, - gboolean RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set); - gboolean NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set); - --/* *INDENT-OFF* */ --enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = { --/* Current State */ --/* Next State: Unknown Stopped Started Slave Master */ -+/* This array says what the *next* role should be when transitioning from one -+ * role to another. For example going from Stopped to Master, the next role is -+ * RSC_ROLE_SLAVE, because the resource must be started before being promoted. -+ * The current state then becomes Started, which is fed into this array again, -+ * giving a next role of RSC_ROLE_MASTER. -+ */ -+static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = { -+ /* Current state Next state*/ -+ /* Unknown Stopped Started Slave Master */ - /* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, }, - /* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, }, - /* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, }, -- /* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, }, -- /* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, }, -+ /* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, }, -+ /* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, }, - }; - --gboolean (*rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX])(pe_resource_t*,pe_node_t*,gboolean,pe_working_set_t*) = { --/* Current State */ --/* Next State: Unknown Stopped Started Slave Master */ -- /* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, }, -- /* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, }, -- /* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, }, -- /* Slave */ { RoleError, StopRsc, StopRsc, NullOp, PromoteRsc, }, -- /* Master */ { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp, }, -+typedef gboolean (*rsc_transition_fn)(pe_resource_t *rsc, pe_node_t *next, -+ gboolean optional, -+ pe_working_set_t *data_set); -+ -+// This array picks the function needed to transition from one role to another -+static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = { -+ /* Current state Next state */ -+ /* Unknown Stopped Started Slave Master */ -+ /* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, }, -+ /* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, }, -+ /* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, }, -+ /* Slave */ { RoleError, StopRsc, StopRsc, NullOp, PromoteRsc, }, -+ /* Master */ { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp , }, - }; --/* *INDENT-ON* */ - - static gboolean - native_choose_node(pe_resource_t * rsc, pe_node_t * prefer, pe_working_set_t * data_set) --- -1.8.3.1 - - -From 2f1e2df1f5ec67591cddf14f9dda1c52919dd53a Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 26 May 2020 17:50:48 -0500 -Subject: [PATCH 09/20] Feature: xml: add on-fail="demote" option to resources - schema - -We don't need an XML schema version bump because it was already bumped since -the last release, for the rsc_expression/op_expression feature. ---- - xml/resources-3.4.rng | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/xml/resources-3.4.rng b/xml/resources-3.4.rng -index fbb4b65..887dc1c 100644 ---- a/xml/resources-3.4.rng -+++ b/xml/resources-3.4.rng -@@ -388,6 +388,7 @@ - - ignore - block -+ demote - stop - restart - standby --- -1.8.3.1 - - -From 874f75e0faad91c634860221d727e51e95d97f19 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 28 May 2020 08:29:37 -0500 -Subject: [PATCH 10/20] Feature: scheduler: new on-fail="demote" recovery - policy for promoted resources - ---- - include/crm/pengine/pe_types.h | 1 + - lib/pacemaker/pcmk_sched_native.c | 25 +++++++++++++++---- - lib/pengine/common.c | 3 +++ - lib/pengine/unpack.c | 51 ++++++++++++++++++++++++++++++++++++--- - lib/pengine/utils.c | 35 +++++++++++++++++++++++---- - 5 files changed, 102 insertions(+), 13 deletions(-) - -diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h -index ba88491..ed5eb12 100644 ---- a/include/crm/pengine/pe_types.h -+++ b/include/crm/pengine/pe_types.h -@@ -246,6 +246,7 @@ struct pe_node_s { - # define pe_rsc_allocating 0x00000200ULL - # define pe_rsc_merging 0x00000400ULL - -+# define pe_rsc_stop 0x00001000ULL - # define pe_rsc_reload 0x00002000ULL - # define pe_rsc_allow_remote_remotes 0x00004000ULL - -diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c -index b9bca80..4e3bd7c 100644 ---- a/lib/pacemaker/pcmk_sched_native.c -+++ b/lib/pacemaker/pcmk_sched_native.c -@@ -1205,6 +1205,7 @@ native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set) - pe_node_t *chosen = NULL; - pe_node_t *current = NULL; - gboolean need_stop = FALSE; -+ bool need_promote = FALSE; - gboolean is_moving = FALSE; - gboolean allow_migrate = is_set(rsc->flags, pe_rsc_allow_migrate) ? TRUE : FALSE; - -@@ -1309,8 +1310,15 @@ native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set) - need_stop = TRUE; - - } else if (is_set(rsc->flags, pe_rsc_failed)) { -- pe_rsc_trace(rsc, "Recovering %s", rsc->id); -- need_stop = TRUE; -+ if (is_set(rsc->flags, pe_rsc_stop)) { -+ need_stop = TRUE; -+ pe_rsc_trace(rsc, "Recovering %s", rsc->id); -+ } else { -+ pe_rsc_trace(rsc, "Recovering %s by demotion", rsc->id); -+ if (rsc->next_role == RSC_ROLE_MASTER) { -+ need_promote = TRUE; -+ } -+ } - - } else if (is_set(rsc->flags, pe_rsc_block)) { - pe_rsc_trace(rsc, "Block %s", rsc->id); -@@ -1344,10 +1352,16 @@ native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set) - - - while (rsc->role <= rsc->next_role && role != rsc->role && is_not_set(rsc->flags, pe_rsc_block)) { -+ bool required = need_stop; -+ - next_role = rsc_state_matrix[role][rsc->role]; -+ if ((next_role == RSC_ROLE_MASTER) && need_promote) { -+ required = true; -+ } - pe_rsc_trace(rsc, "Up: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role), -- rsc->id, need_stop ? " required" : ""); -- if (rsc_action_matrix[role][next_role] (rsc, chosen, !need_stop, data_set) == FALSE) { -+ rsc->id, (required? " required" : "")); -+ if (rsc_action_matrix[role][next_role](rsc, chosen, !required, -+ data_set) == FALSE) { - break; - } - role = next_role; -@@ -2631,7 +2645,8 @@ LogActions(pe_resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) - - free(key); - -- } else if (stop && is_set(rsc->flags, pe_rsc_failed)) { -+ } else if (stop && is_set(rsc->flags, pe_rsc_failed) -+ && is_set(rsc->flags, pe_rsc_stop)) { - /* 'stop' may be NULL if the failure was ignored */ - LogAction("Recover", rsc, current, next, stop, start, terminal); - STOP_SANITY_ASSERT(__LINE__); -diff --git a/lib/pengine/common.c b/lib/pengine/common.c -index ded6df8..f4f2106 100644 ---- a/lib/pengine/common.c -+++ b/lib/pengine/common.c -@@ -326,6 +326,9 @@ fail2text(enum action_fail_response fail) - case action_fail_ignore: - result = "ignore"; - break; -+ case action_fail_demote: -+ result = "demote"; -+ break; - case action_fail_block: - result = "block"; - break; -diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c -index 6a350e5..a219805 100644 ---- a/lib/pengine/unpack.c -+++ b/lib/pengine/unpack.c -@@ -108,6 +108,7 @@ pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, - */ - node->details->remote_requires_reset = TRUE; - set_bit(rsc->flags, pe_rsc_failed); -+ set_bit(rsc->flags, pe_rsc_stop); - } - } - -@@ -117,6 +118,7 @@ pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, - "and guest resource no longer exists", - node->details->uname, reason); - set_bit(node->details->remote_rsc->flags, pe_rsc_failed); -+ set_bit(node->details->remote_rsc->flags, pe_rsc_stop); - - } else if (pe__is_remote_node(node)) { - pe_resource_t *rsc = node->details->remote_rsc; -@@ -1914,6 +1916,7 @@ process_rsc_state(pe_resource_t * rsc, pe_node_t * node, - */ - if (pe__is_guest_node(node)) { - set_bit(rsc->flags, pe_rsc_failed); -+ set_bit(rsc->flags, pe_rsc_stop); - should_fence = TRUE; - - } else if (is_set(data_set->flags, pe_flag_stonith_enabled)) { -@@ -1956,6 +1959,11 @@ process_rsc_state(pe_resource_t * rsc, pe_node_t * node, - /* nothing to do */ - break; - -+ case action_fail_demote: -+ set_bit(rsc->flags, pe_rsc_failed); -+ demote_action(rsc, node, FALSE); -+ break; -+ - case action_fail_fence: - /* treat it as if it is still running - * but also mark the node as unclean -@@ -1992,12 +2000,14 @@ process_rsc_state(pe_resource_t * rsc, pe_node_t * node, - case action_fail_recover: - if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) { - set_bit(rsc->flags, pe_rsc_failed); -+ set_bit(rsc->flags, pe_rsc_stop); - stop_action(rsc, node, FALSE); - } - break; - - case action_fail_restart_container: - set_bit(rsc->flags, pe_rsc_failed); -+ set_bit(rsc->flags, pe_rsc_stop); - - if (rsc->container && pe_rsc_is_bundled(rsc)) { - /* A bundle's remote connection can run on a different node than -@@ -2016,6 +2026,7 @@ process_rsc_state(pe_resource_t * rsc, pe_node_t * node, - - case action_fail_reset_remote: - set_bit(rsc->flags, pe_rsc_failed); -+ set_bit(rsc->flags, pe_rsc_stop); - if (is_set(data_set->flags, pe_flag_stonith_enabled)) { - tmpnode = NULL; - if (rsc->is_remote_node) { -@@ -2071,8 +2082,17 @@ process_rsc_state(pe_resource_t * rsc, pe_node_t * node, - } - - native_add_running(rsc, node, data_set); -- if (on_fail != action_fail_ignore) { -- set_bit(rsc->flags, pe_rsc_failed); -+ switch (on_fail) { -+ case action_fail_ignore: -+ break; -+ case action_fail_demote: -+ case action_fail_block: -+ set_bit(rsc->flags, pe_rsc_failed); -+ break; -+ default: -+ set_bit(rsc->flags, pe_rsc_failed); -+ set_bit(rsc->flags, pe_rsc_stop); -+ break; - } - - } else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) { -@@ -2595,6 +2615,7 @@ unpack_migrate_to_success(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op, - } else { - /* Consider it failed here - forces a restart, prevents migration */ - set_bit(rsc->flags, pe_rsc_failed); -+ set_bit(rsc->flags, pe_rsc_stop); - clear_bit(rsc->flags, pe_rsc_allow_migrate); - } - } -@@ -2785,9 +2806,21 @@ static int - cmp_on_fail(enum action_fail_response first, enum action_fail_response second) - { - switch (first) { -+ case action_fail_demote: -+ switch (second) { -+ case action_fail_ignore: -+ return 1; -+ case action_fail_demote: -+ return 0; -+ default: -+ return -1; -+ } -+ break; -+ - case action_fail_reset_remote: - switch (second) { - case action_fail_ignore: -+ case action_fail_demote: - case action_fail_recover: - return 1; - case action_fail_reset_remote: -@@ -2800,6 +2833,7 @@ cmp_on_fail(enum action_fail_response first, enum action_fail_response second) - case action_fail_restart_container: - switch (second) { - case action_fail_ignore: -+ case action_fail_demote: - case action_fail_recover: - case action_fail_reset_remote: - return 1; -@@ -2814,9 +2848,13 @@ cmp_on_fail(enum action_fail_response first, enum action_fail_response second) - break; - } - switch (second) { -+ case action_fail_demote: -+ return (first == action_fail_ignore)? -1 : 1; -+ - case action_fail_reset_remote: - switch (first) { - case action_fail_ignore: -+ case action_fail_demote: - case action_fail_recover: - return -1; - default: -@@ -2827,6 +2865,7 @@ cmp_on_fail(enum action_fail_response first, enum action_fail_response second) - case action_fail_restart_container: - switch (first) { - case action_fail_ignore: -+ case action_fail_demote: - case action_fail_recover: - case action_fail_reset_remote: - return -1; -@@ -3426,7 +3465,11 @@ update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, c - clear_past_failure = TRUE; - - } else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) { -- /* Demote from Master does not clear an error */ -+ -+ if (*on_fail == action_fail_demote) { -+ // Demote clears an error only if on-fail=demote -+ clear_past_failure = TRUE; -+ } - rsc->role = RSC_ROLE_SLAVE; - - } else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) { -@@ -3454,6 +3497,7 @@ update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, c - - case action_fail_block: - case action_fail_ignore: -+ case action_fail_demote: - case action_fail_recover: - case action_fail_restart_container: - *on_fail = action_fail_ignore; -@@ -3714,6 +3758,7 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op, - * that, ensure the remote connection is considered failed. - */ - set_bit(node->details->remote_rsc->flags, pe_rsc_failed); -+ set_bit(node->details->remote_rsc->flags, pe_rsc_stop); - } - - // fall through -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index 3fb7e62..fee9efb 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -720,6 +720,7 @@ static bool - valid_stop_on_fail(const char *value) - { - return safe_str_neq(value, "standby") -+ && safe_str_neq(value, "demote") - && safe_str_neq(value, "stop"); - } - -@@ -727,6 +728,11 @@ static const char * - unpack_operation_on_fail(pe_action_t * action) - { - -+ const char *name = NULL; -+ const char *role = NULL; -+ const char *on_fail = NULL; -+ const char *interval_spec = NULL; -+ const char *enabled = NULL; - const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL); - - if (safe_str_eq(action->task, CRMD_ACTION_STOP) -@@ -736,14 +742,10 @@ unpack_operation_on_fail(pe_action_t * action) - "action to default value because '%s' is not " - "allowed for stop", action->rsc->id, value); - return NULL; -+ - } else if (safe_str_eq(action->task, CRMD_ACTION_DEMOTE) && !value) { - /* demote on_fail defaults to master monitor value if present */ - xmlNode *operation = NULL; -- const char *name = NULL; -- const char *role = NULL; -- const char *on_fail = NULL; -- const char *interval_spec = NULL; -- const char *enabled = NULL; - - CRM_CHECK(action->rsc != NULL, return NULL); - -@@ -766,12 +768,31 @@ unpack_operation_on_fail(pe_action_t * action) - continue; - } else if (crm_parse_interval_spec(interval_spec) == 0) { - continue; -+ } else if (safe_str_eq(on_fail, "demote")) { -+ continue; - } - - value = on_fail; - } - } else if (safe_str_eq(action->task, CRM_OP_LRM_DELETE)) { - value = "ignore"; -+ -+ } else if (safe_str_eq(value, "demote")) { -+ name = crm_element_value(action->op_entry, "name"); -+ role = crm_element_value(action->op_entry, "role"); -+ on_fail = crm_element_value(action->op_entry, XML_OP_ATTR_ON_FAIL); -+ interval_spec = crm_element_value(action->op_entry, -+ XML_LRM_ATTR_INTERVAL); -+ -+ if (safe_str_neq(name, CRMD_ACTION_PROMOTE) -+ && (safe_str_neq(name, CRMD_ACTION_STATUS) -+ || safe_str_neq(role, "Master") -+ || (crm_parse_interval_spec(interval_spec) == 0))) { -+ pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s %s " -+ "action to default value because 'demote' is not " -+ "allowed for it", action->rsc->id, name); -+ return NULL; -+ } - } - - return value; -@@ -1170,6 +1191,10 @@ unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * contai - value = NULL; - } - -+ } else if (safe_str_eq(value, "demote")) { -+ action->on_fail = action_fail_demote; -+ value = "demote instance"; -+ - } else { - pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value); - value = NULL; --- -1.8.3.1 - - -From d29433ea57796de000f4fea8c60f8da1d903108b Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 16 Jun 2020 16:03:14 -0500 -Subject: [PATCH 11/20] Test: scheduler: add regression tests for - on-fail="demote" - ---- - cts/cts-scheduler.in | 4 + - cts/scheduler/on_fail_demote1.dot | 64 ++ - cts/scheduler/on_fail_demote1.exp | 360 +++++++ - cts/scheduler/on_fail_demote1.scores | 470 +++++++++ - cts/scheduler/on_fail_demote1.summary | 86 ++ - cts/scheduler/on_fail_demote1.xml | 616 +++++++++++ - cts/scheduler/on_fail_demote2.dot | 22 + - cts/scheduler/on_fail_demote2.exp | 125 +++ - cts/scheduler/on_fail_demote2.scores | 127 +++ - cts/scheduler/on_fail_demote2.summary | 41 + - cts/scheduler/on_fail_demote2.xml | 221 ++++ - cts/scheduler/on_fail_demote3.dot | 12 + - cts/scheduler/on_fail_demote3.exp | 63 ++ - cts/scheduler/on_fail_demote3.scores | 127 +++ - cts/scheduler/on_fail_demote3.summary | 34 + - cts/scheduler/on_fail_demote3.xml | 221 ++++ - cts/scheduler/on_fail_demote4.dot | 383 +++++++ - cts/scheduler/on_fail_demote4.exp | 1818 +++++++++++++++++++++++++++++++++ - cts/scheduler/on_fail_demote4.scores | 470 +++++++++ - cts/scheduler/on_fail_demote4.summary | 187 ++++ - cts/scheduler/on_fail_demote4.xml | 625 ++++++++++++ - 21 files changed, 6076 insertions(+) - create mode 100644 cts/scheduler/on_fail_demote1.dot - create mode 100644 cts/scheduler/on_fail_demote1.exp - create mode 100644 cts/scheduler/on_fail_demote1.scores - create mode 100644 cts/scheduler/on_fail_demote1.summary - create mode 100644 cts/scheduler/on_fail_demote1.xml - create mode 100644 cts/scheduler/on_fail_demote2.dot - create mode 100644 cts/scheduler/on_fail_demote2.exp - create mode 100644 cts/scheduler/on_fail_demote2.scores - create mode 100644 cts/scheduler/on_fail_demote2.summary - create mode 100644 cts/scheduler/on_fail_demote2.xml - create mode 100644 cts/scheduler/on_fail_demote3.dot - create mode 100644 cts/scheduler/on_fail_demote3.exp - create mode 100644 cts/scheduler/on_fail_demote3.scores - create mode 100644 cts/scheduler/on_fail_demote3.summary - create mode 100644 cts/scheduler/on_fail_demote3.xml - create mode 100644 cts/scheduler/on_fail_demote4.dot - create mode 100644 cts/scheduler/on_fail_demote4.exp - create mode 100644 cts/scheduler/on_fail_demote4.scores - create mode 100644 cts/scheduler/on_fail_demote4.summary - create mode 100644 cts/scheduler/on_fail_demote4.xml - -diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in -index ae8247e..0e68e73 100644 ---- a/cts/cts-scheduler.in -+++ b/cts/cts-scheduler.in -@@ -478,6 +478,10 @@ TESTS = [ - [ "master-score-startup", "Use permanent master scores without LRM history" ], - [ "failed-demote-recovery", "Recover resource in slave role after demote fails" ], - [ "failed-demote-recovery-master", "Recover resource in master role after demote fails" ], -+ [ "on_fail_demote1", "Recovery with on-fail=\"demote\" on healthy cluster, remote, guest, and bundle nodes" ], -+ [ "on_fail_demote2", "Recovery with on-fail=\"demote\" with promotion on different node" ], -+ [ "on_fail_demote3", "Recovery with on-fail=\"demote\" with no promotion" ], -+ [ "on_fail_demote4", "Recovery with on-fail=\"demote\" on failed cluster, remote, guest, and bundle nodes" ], - ], - [ - [ "history-1", "Correctly parse stateful-1 resource state" ], -diff --git a/cts/scheduler/on_fail_demote1.dot b/cts/scheduler/on_fail_demote1.dot -new file mode 100644 -index 0000000..d11c1c1 ---- /dev/null -+++ b/cts/scheduler/on_fail_demote1.dot -@@ -0,0 +1,64 @@ -+ digraph "g" { -+"bundled_demote_0 stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"bundled_demote_0 stateful-bundle-0" -> "stateful-bundle-master_demoted_0" [ style = bold] -+"bundled_demote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"] -+"bundled_promote_0 stateful-bundle-0" -> "stateful-bundle-master_promoted_0" [ style = bold] -+"bundled_promote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"] -+"lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold] -+"lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc2" [ style = bold] -+"lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold] -+"lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms_demote_0 lxc2" -> "lxc-ms-master_demoted_0" [ style = bold] -+"lxc-ms_demote_0 lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"lxc-ms_demote_0 lxc2" [ style=bold color="green" fontcolor="black"] -+"lxc-ms_promote_0 lxc2" -> "lxc-ms-master_promoted_0" [ style = bold] -+"lxc-ms_promote_0 lxc2" [ style=bold color="green" fontcolor="black"] -+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold] -+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold] -+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_demoted_0" -> "rsc1-clone_promote_0" [ style = bold] -+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_promote_0" -> "rsc1_promote_0 rhel7-4" [ style = bold] -+"rsc1-clone_promote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold] -+"rsc1_demote_0 rhel7-4" -> "rsc1_promote_0 rhel7-4" [ style = bold] -+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="black"] -+"rsc1_promote_0 rhel7-4" -> "rsc1-clone_promoted_0" [ style = bold] -+"rsc1_promote_0 rhel7-4" [ style=bold color="green" fontcolor="black"] -+"rsc2-master_demote_0" -> "rsc2-master_demoted_0" [ style = bold] -+"rsc2-master_demote_0" -> "rsc2_demote_0 remote-rhel7-2" [ style = bold] -+"rsc2-master_demote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2-master_demoted_0" -> "rsc2-master_promote_0" [ style = bold] -+"rsc2-master_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2-master_promote_0" -> "rsc2_promote_0 remote-rhel7-2" [ style = bold] -+"rsc2-master_promote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2-master_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2_demote_0 remote-rhel7-2" -> "rsc2-master_demoted_0" [ style = bold] -+"rsc2_demote_0 remote-rhel7-2" -> "rsc2_promote_0 remote-rhel7-2" [ style = bold] -+"rsc2_demote_0 remote-rhel7-2" [ style=bold color="green" fontcolor="black"] -+"rsc2_promote_0 remote-rhel7-2" -> "rsc2-master_promoted_0" [ style = bold] -+"rsc2_promote_0 remote-rhel7-2" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-master_demote_0" -> "bundled_demote_0 stateful-bundle-0" [ style = bold] -+"stateful-bundle-master_demote_0" -> "stateful-bundle-master_demoted_0" [ style = bold] -+"stateful-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_promote_0" [ style = bold] -+"stateful-bundle-master_demoted_0" -> "stateful-bundle_demoted_0" [ style = bold] -+"stateful-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-master_promote_0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"stateful-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-master_promoted_0" -> "stateful-bundle_promoted_0" [ style = bold] -+"stateful-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_demote_0" -> "stateful-bundle-master_demote_0" [ style = bold] -+"stateful-bundle_demote_0" -> "stateful-bundle_demoted_0" [ style = bold] -+"stateful-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_demoted_0" -> "stateful-bundle_promote_0" [ style = bold] -+"stateful-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_promote_0" -> "stateful-bundle-master_promote_0" [ style = bold] -+"stateful-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] -+} -diff --git a/cts/scheduler/on_fail_demote1.exp b/cts/scheduler/on_fail_demote1.exp -new file mode 100644 -index 0000000..ebe1dd5 ---- /dev/null -+++ b/cts/scheduler/on_fail_demote1.exp -@@ -0,0 +1,360 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/on_fail_demote1.scores b/cts/scheduler/on_fail_demote1.scores -new file mode 100644 -index 0000000..7df582f ---- /dev/null -+++ b/cts/scheduler/on_fail_demote1.scores -@@ -0,0 +1,470 @@ -+Allocation scores: -+Using the original execution date of: 2020-06-16 19:23:21Z -+bundled:0 promotion score on stateful-bundle-0: 10 -+bundled:1 promotion score on stateful-bundle-1: 5 -+bundled:2 promotion score on stateful-bundle-2: 5 -+lxc-ms:0 promotion score on lxc2: INFINITY -+lxc-ms:1 promotion score on lxc1: INFINITY -+pcmk__bundle_allocate: bundled:0 allocation score on stateful-bundle-0: 501 -+pcmk__bundle_allocate: bundled:1 allocation score on stateful-bundle-1: 501 -+pcmk__bundle_allocate: bundled:2 allocation score on stateful-bundle-2: 501 -+pcmk__bundle_allocate: stateful-bundle allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-0: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-1: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-2: -INFINITY -+pcmk__clone_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY -+pcmk__clone_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY -+pcmk__clone_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY -+pcmk__clone_allocate: lxc-ms-master allocation score on lxc1: INFINITY -+pcmk__clone_allocate: lxc-ms-master allocation score on lxc2: INFINITY -+pcmk__clone_allocate: lxc-ms-master allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-1: 0 -+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-3: 0 -+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-4: 0 -+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-5: 0 -+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc1: INFINITY -+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc2: INFINITY -+pcmk__clone_allocate: lxc-ms:0 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc1: INFINITY -+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc2: INFINITY -+pcmk__clone_allocate: lxc-ms:1 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 11 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6 -+pcmk__clone_allocate: rsc1:3 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on remote-rhel7-2: 6 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on lxc2: 6 -+pcmk__clone_allocate: rsc1:5 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on lxc1: 6 -+pcmk__clone_allocate: rsc1:6 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2-master allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2-master allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2-master allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 11 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6 -+pcmk__clone_allocate: rsc2:3 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on remote-rhel7-2: 11 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on lxc2: 6 -+pcmk__clone_allocate: rsc2:5 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on lxc1: 6 -+pcmk__clone_allocate: rsc2:6 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc1: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc2: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on remote-rhel7-2: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-1: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-3: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-4: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-5: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-0: 0 -+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-1: 0 -+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-2: 0 -+pcmk__native_allocate: Fencing allocation score on lxc1: -INFINITY -+pcmk__native_allocate: Fencing allocation score on lxc2: -INFINITY -+pcmk__native_allocate: Fencing allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0 -+pcmk__native_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY -+pcmk__native_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY -+pcmk__native_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY -+pcmk__native_allocate: container1 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: container1 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: container1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: container1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: container1 allocation score on rhel7-3: INFINITY -+pcmk__native_allocate: container1 allocation score on rhel7-4: 0 -+pcmk__native_allocate: container1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: container2 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: container2 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: container2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: container2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: container2 allocation score on rhel7-3: INFINITY -+pcmk__native_allocate: container2 allocation score on rhel7-4: 0 -+pcmk__native_allocate: container2 allocation score on rhel7-5: 0 -+pcmk__native_allocate: lxc-ms:0 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: lxc-ms:0 allocation score on lxc2: INFINITY -+pcmk__native_allocate: lxc-ms:0 allocation score on remote-rhel7-2: 0 -+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-1: 0 -+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-3: 0 -+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-4: 0 -+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-5: 0 -+pcmk__native_allocate: lxc-ms:1 allocation score on lxc1: INFINITY -+pcmk__native_allocate: lxc-ms:1 allocation score on lxc2: INFINITY -+pcmk__native_allocate: lxc-ms:1 allocation score on remote-rhel7-2: 0 -+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-3: 0 -+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-4: 0 -+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: lxc1 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: lxc1 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: lxc1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: lxc1 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: lxc1 allocation score on rhel7-3: 0 -+pcmk__native_allocate: lxc1 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: lxc1 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on rhel7-3: 0 -+pcmk__native_allocate: lxc2 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: remote-rhel7-2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-3: 0 -+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-4: 0 -+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc1:0 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: 11 -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:1 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc1:1 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc1:1 allocation score on remote-rhel7-2: 0 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc1:2 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc1:2 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc1:2 allocation score on remote-rhel7-2: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6 -+pcmk__native_allocate: rsc1:3 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc1:3 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc1:3 allocation score on remote-rhel7-2: 0 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc1:4 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc1:4 allocation score on remote-rhel7-2: 6 -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:5 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc1:5 allocation score on lxc2: 6 -+pcmk__native_allocate: rsc1:5 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:5 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:5 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:5 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:5 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:6 allocation score on lxc1: 6 -+pcmk__native_allocate: rsc1:6 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: rsc1:6 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:6 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:6 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:6 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:6 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:0 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc2:0 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc2:0 allocation score on remote-rhel7-2: 0 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: 0 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: 11 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc2:1 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc2:1 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc2:1 allocation score on remote-rhel7-2: 0 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc2:2 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc2:2 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc2:2 allocation score on remote-rhel7-2: 0 -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6 -+pcmk__native_allocate: rsc2:3 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc2:3 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc2:3 allocation score on remote-rhel7-2: 0 -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6 -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on remote-rhel7-2: 11 -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:5 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc2:5 allocation score on lxc2: 6 -+pcmk__native_allocate: rsc2:5 allocation score on remote-rhel7-2: 0 -+pcmk__native_allocate: rsc2:5 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc2:5 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:5 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:5 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:6 allocation score on lxc1: 6 -+pcmk__native_allocate: rsc2:6 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: rsc2:6 allocation score on remote-rhel7-2: 0 -+pcmk__native_allocate: rsc2:6 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc2:6 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:6 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:6 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-1: 0 -+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-4: 0 -+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-5: 10000 -+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-1: 10000 -+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-4: 0 -+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-4: 10000 -+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-5: 0 -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc1: -10000 -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc2: -10000 -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: -10000 -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: 0 -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: 0 -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc1: -10000 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc2: -10000 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: -10000 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: 0 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc1: -10000 -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc2: -10000 -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: -10000 -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: 0 -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0 -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0 -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: 0 -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: -INFINITY -+rsc1:0 promotion score on rhel7-4: 10 -+rsc1:1 promotion score on rhel7-3: 5 -+rsc1:2 promotion score on rhel7-5: 5 -+rsc1:3 promotion score on rhel7-1: 5 -+rsc1:4 promotion score on remote-rhel7-2: 5 -+rsc1:5 promotion score on lxc2: 5 -+rsc1:6 promotion score on lxc1: 5 -+rsc2:0 promotion score on rhel7-4: 10 -+rsc2:1 promotion score on rhel7-3: 5 -+rsc2:2 promotion score on rhel7-5: 5 -+rsc2:3 promotion score on rhel7-1: 5 -+rsc2:4 promotion score on remote-rhel7-2: 110 -+rsc2:5 promotion score on lxc2: 5 -+rsc2:6 promotion score on lxc1: 5 -diff --git a/cts/scheduler/on_fail_demote1.summary b/cts/scheduler/on_fail_demote1.summary -new file mode 100644 -index 0000000..b173582 ---- /dev/null -+++ b/cts/scheduler/on_fail_demote1.summary -@@ -0,0 +1,86 @@ -+Using the original execution date of: 2020-06-16 19:23:21Z -+ -+Current cluster status: -+Online: [ rhel7-1 rhel7-3 rhel7-4 rhel7-5 ] -+RemoteOnline: [ remote-rhel7-2 ] -+GuestOnline: [ lxc1:container1 lxc2:container2 stateful-bundle-0:stateful-bundle-docker-0 stateful-bundle-1:stateful-bundle-docker-1 stateful-bundle-2:stateful-bundle-docker-2 ] -+ -+ Fencing (stonith:fence_xvm): Started rhel7-4 -+ Clone Set: rsc1-clone [rsc1] (promotable) -+ rsc1 (ocf::pacemaker:Stateful): FAILED Master rhel7-4 -+ Slaves: [ lxc1 lxc2 remote-rhel7-2 rhel7-1 rhel7-3 rhel7-5 ] -+ Clone Set: rsc2-master [rsc2] (promotable) -+ rsc2 (ocf::pacemaker:Stateful): FAILED Master remote-rhel7-2 -+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ] -+ remote-rhel7-2 (ocf::pacemaker:remote): Started rhel7-1 -+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3 -+ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-3 -+ Clone Set: lxc-ms-master [lxc-ms] (promotable) -+ lxc-ms (ocf::pacemaker:Stateful): FAILED Master lxc2 -+ Slaves: [ lxc1 ] -+ Stopped: [ remote-rhel7-2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ] -+ Container bundle set: stateful-bundle [pcmktest:http] -+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): FAILED Master rhel7-5 -+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1 -+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): Slave rhel7-4 -+ -+Transition Summary: -+ * Re-promote rsc1:0 ( Master rhel7-4 ) -+ * Re-promote rsc2:4 ( Master remote-rhel7-2 ) -+ * Re-promote lxc-ms:0 ( Master lxc2 ) -+ * Re-promote bundled:0 ( Master stateful-bundle-0 ) -+ -+Executing cluster transition: -+ * Pseudo action: rsc1-clone_demote_0 -+ * Pseudo action: rsc2-master_demote_0 -+ * Pseudo action: lxc-ms-master_demote_0 -+ * Pseudo action: stateful-bundle_demote_0 -+ * Resource action: rsc1 demote on rhel7-4 -+ * Pseudo action: rsc1-clone_demoted_0 -+ * Pseudo action: rsc1-clone_promote_0 -+ * Resource action: rsc2 demote on remote-rhel7-2 -+ * Pseudo action: rsc2-master_demoted_0 -+ * Pseudo action: rsc2-master_promote_0 -+ * Resource action: lxc-ms demote on lxc2 -+ * Pseudo action: lxc-ms-master_demoted_0 -+ * Pseudo action: lxc-ms-master_promote_0 -+ * Pseudo action: stateful-bundle-master_demote_0 -+ * Resource action: rsc1 promote on rhel7-4 -+ * Pseudo action: rsc1-clone_promoted_0 -+ * Resource action: rsc2 promote on remote-rhel7-2 -+ * Pseudo action: rsc2-master_promoted_0 -+ * Resource action: lxc-ms promote on lxc2 -+ * Pseudo action: lxc-ms-master_promoted_0 -+ * Resource action: bundled demote on stateful-bundle-0 -+ * Pseudo action: stateful-bundle-master_demoted_0 -+ * Pseudo action: stateful-bundle_demoted_0 -+ * Pseudo action: stateful-bundle_promote_0 -+ * Pseudo action: stateful-bundle-master_promote_0 -+ * Resource action: bundled promote on stateful-bundle-0 -+ * Pseudo action: stateful-bundle-master_promoted_0 -+ * Pseudo action: stateful-bundle_promoted_0 -+Using the original execution date of: 2020-06-16 19:23:21Z -+ -+Revised cluster status: -+Online: [ rhel7-1 rhel7-3 rhel7-4 rhel7-5 ] -+RemoteOnline: [ remote-rhel7-2 ] -+GuestOnline: [ lxc1:container1 lxc2:container2 stateful-bundle-0:stateful-bundle-docker-0 stateful-bundle-1:stateful-bundle-docker-1 stateful-bundle-2:stateful-bundle-docker-2 ] -+ -+ Fencing (stonith:fence_xvm): Started rhel7-4 -+ Clone Set: rsc1-clone [rsc1] (promotable) -+ Masters: [ rhel7-4 ] -+ Slaves: [ lxc1 lxc2 remote-rhel7-2 rhel7-1 rhel7-3 rhel7-5 ] -+ Clone Set: rsc2-master [rsc2] (promotable) -+ Masters: [ remote-rhel7-2 ] -+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ] -+ remote-rhel7-2 (ocf::pacemaker:remote): Started rhel7-1 -+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3 -+ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-3 -+ Clone Set: lxc-ms-master [lxc-ms] (promotable) -+ Masters: [ lxc2 ] -+ Slaves: [ lxc1 ] -+ Container bundle set: stateful-bundle [pcmktest:http] -+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): Master rhel7-5 -+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1 -+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): Slave rhel7-4 -+ -diff --git a/cts/scheduler/on_fail_demote1.xml b/cts/scheduler/on_fail_demote1.xml -new file mode 100644 -index 0000000..9f3ff20 ---- /dev/null -+++ b/cts/scheduler/on_fail_demote1.xml -@@ -0,0 +1,616 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/on_fail_demote2.dot b/cts/scheduler/on_fail_demote2.dot -new file mode 100644 -index 0000000..06193cb ---- /dev/null -+++ b/cts/scheduler/on_fail_demote2.dot -@@ -0,0 +1,22 @@ -+ digraph "g" { -+"Cancel rsc1_monitor_10000 rhel7-4" -> "rsc1_demote_0 rhel7-4" [ style = bold] -+"Cancel rsc1_monitor_10000 rhel7-4" [ style=bold color="green" fontcolor="black"] -+"Cancel rsc1_monitor_11000 rhel7-3" -> "rsc1_promote_0 rhel7-3" [ style = bold] -+"Cancel rsc1_monitor_11000 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold] -+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold] -+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_demoted_0" -> "rsc1-clone_promote_0" [ style = bold] -+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_promote_0" -> "rsc1_promote_0 rhel7-3" [ style = bold] -+"rsc1-clone_promote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold] -+"rsc1_demote_0 rhel7-4" -> "rsc1_monitor_11000 rhel7-4" [ style = bold] -+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="black"] -+"rsc1_monitor_10000 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"rsc1_monitor_11000 rhel7-4" [ style=bold color="green" fontcolor="black"] -+"rsc1_promote_0 rhel7-3" -> "rsc1-clone_promoted_0" [ style = bold] -+"rsc1_promote_0 rhel7-3" -> "rsc1_monitor_10000 rhel7-3" [ style = bold] -+"rsc1_promote_0 rhel7-3" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/on_fail_demote2.exp b/cts/scheduler/on_fail_demote2.exp -new file mode 100644 -index 0000000..492e86f ---- /dev/null -+++ b/cts/scheduler/on_fail_demote2.exp -@@ -0,0 +1,125 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/on_fail_demote2.scores b/cts/scheduler/on_fail_demote2.scores -new file mode 100644 -index 0000000..25aea90 ---- /dev/null -+++ b/cts/scheduler/on_fail_demote2.scores -@@ -0,0 +1,127 @@ -+Allocation scores: -+Using the original execution date of: 2020-06-16 19:23:21Z -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 11 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-2: 6 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 11 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-2: 6 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-2: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: 11 -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-2: 6 -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: 0 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: 11 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6 -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6 -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-2: 6 -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY -+rsc1:0 promotion score on rhel7-4: -INFINITY -+rsc1:1 promotion score on rhel7-3: 5 -+rsc1:2 promotion score on rhel7-5: 5 -+rsc1:3 promotion score on rhel7-1: 5 -+rsc1:4 promotion score on rhel7-2: 5 -+rsc2:0 promotion score on rhel7-4: 10 -+rsc2:1 promotion score on rhel7-3: 5 -+rsc2:2 promotion score on rhel7-5: 5 -+rsc2:3 promotion score on rhel7-1: 5 -+rsc2:4 promotion score on rhel7-2: 5 -diff --git a/cts/scheduler/on_fail_demote2.summary b/cts/scheduler/on_fail_demote2.summary -new file mode 100644 -index 0000000..795a11d ---- /dev/null -+++ b/cts/scheduler/on_fail_demote2.summary -@@ -0,0 +1,41 @@ -+Using the original execution date of: 2020-06-16 19:23:21Z -+ -+Current cluster status: -+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] -+ -+ Fencing (stonith:fence_xvm): Started rhel7-1 -+ Clone Set: rsc1-clone [rsc1] (promotable) -+ rsc1 (ocf::pacemaker:Stateful): FAILED Master rhel7-4 -+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ] -+ Clone Set: rsc2-master [rsc2] (promotable) -+ Masters: [ rhel7-4 ] -+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ] -+ -+Transition Summary: -+ * Demote rsc1:0 ( Master -> Slave rhel7-4 ) -+ * Promote rsc1:1 ( Slave -> Master rhel7-3 ) -+ -+Executing cluster transition: -+ * Resource action: rsc1 cancel=10000 on rhel7-4 -+ * Resource action: rsc1 cancel=11000 on rhel7-3 -+ * Pseudo action: rsc1-clone_demote_0 -+ * Resource action: rsc1 demote on rhel7-4 -+ * Pseudo action: rsc1-clone_demoted_0 -+ * Pseudo action: rsc1-clone_promote_0 -+ * Resource action: rsc1 monitor=11000 on rhel7-4 -+ * Resource action: rsc1 promote on rhel7-3 -+ * Pseudo action: rsc1-clone_promoted_0 -+ * Resource action: rsc1 monitor=10000 on rhel7-3 -+Using the original execution date of: 2020-06-16 19:23:21Z -+ -+Revised cluster status: -+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] -+ -+ Fencing (stonith:fence_xvm): Started rhel7-1 -+ Clone Set: rsc1-clone [rsc1] (promotable) -+ Masters: [ rhel7-3 ] -+ Slaves: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ] -+ Clone Set: rsc2-master [rsc2] (promotable) -+ Masters: [ rhel7-4 ] -+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ] -+ -diff --git a/cts/scheduler/on_fail_demote2.xml b/cts/scheduler/on_fail_demote2.xml -new file mode 100644 -index 0000000..ae91633 ---- /dev/null -+++ b/cts/scheduler/on_fail_demote2.xml -@@ -0,0 +1,221 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/on_fail_demote3.dot b/cts/scheduler/on_fail_demote3.dot -new file mode 100644 -index 0000000..e78325b ---- /dev/null -+++ b/cts/scheduler/on_fail_demote3.dot -@@ -0,0 +1,12 @@ -+ digraph "g" { -+"Cancel rsc1_monitor_10000 rhel7-4" -> "rsc1_demote_0 rhel7-4" [ style = bold] -+"Cancel rsc1_monitor_10000 rhel7-4" [ style=bold color="green" fontcolor="black"] -+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold] -+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold] -+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold] -+"rsc1_demote_0 rhel7-4" -> "rsc1_monitor_11000 rhel7-4" [ style = bold] -+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="black"] -+"rsc1_monitor_11000 rhel7-4" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/on_fail_demote3.exp b/cts/scheduler/on_fail_demote3.exp -new file mode 100644 -index 0000000..ed6bd6d ---- /dev/null -+++ b/cts/scheduler/on_fail_demote3.exp -@@ -0,0 +1,63 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/on_fail_demote3.scores b/cts/scheduler/on_fail_demote3.scores -new file mode 100644 -index 0000000..a85639a ---- /dev/null -+++ b/cts/scheduler/on_fail_demote3.scores -@@ -0,0 +1,127 @@ -+Allocation scores: -+Using the original execution date of: 2020-06-16 19:23:21Z -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 11 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-2: 6 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 11 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-2: 6 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-2: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: 11 -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-2: 6 -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: 0 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: 11 -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6 -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6 -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-2: 6 -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY -+rsc1:0 promotion score on rhel7-4: -INFINITY -+rsc1:1 promotion score on rhel7-3: -INFINITY -+rsc1:2 promotion score on rhel7-5: -INFINITY -+rsc1:3 promotion score on rhel7-1: -INFINITY -+rsc1:4 promotion score on rhel7-2: -INFINITY -+rsc2:0 promotion score on rhel7-4: 10 -+rsc2:1 promotion score on rhel7-3: 5 -+rsc2:2 promotion score on rhel7-5: 5 -+rsc2:3 promotion score on rhel7-1: 5 -+rsc2:4 promotion score on rhel7-2: 5 -diff --git a/cts/scheduler/on_fail_demote3.summary b/cts/scheduler/on_fail_demote3.summary -new file mode 100644 -index 0000000..f1173fd ---- /dev/null -+++ b/cts/scheduler/on_fail_demote3.summary -@@ -0,0 +1,34 @@ -+Using the original execution date of: 2020-06-16 19:23:21Z -+ -+Current cluster status: -+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] -+ -+ Fencing (stonith:fence_xvm): Started rhel7-1 -+ Clone Set: rsc1-clone [rsc1] (promotable) -+ rsc1 (ocf::pacemaker:Stateful): FAILED Master rhel7-4 -+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ] -+ Clone Set: rsc2-master [rsc2] (promotable) -+ Masters: [ rhel7-4 ] -+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ] -+ -+Transition Summary: -+ * Demote rsc1:0 ( Master -> Slave rhel7-4 ) -+ -+Executing cluster transition: -+ * Resource action: rsc1 cancel=10000 on rhel7-4 -+ * Pseudo action: rsc1-clone_demote_0 -+ * Resource action: rsc1 demote on rhel7-4 -+ * Pseudo action: rsc1-clone_demoted_0 -+ * Resource action: rsc1 monitor=11000 on rhel7-4 -+Using the original execution date of: 2020-06-16 19:23:21Z -+ -+Revised cluster status: -+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] -+ -+ Fencing (stonith:fence_xvm): Started rhel7-1 -+ Clone Set: rsc1-clone [rsc1] (promotable) -+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] -+ Clone Set: rsc2-master [rsc2] (promotable) -+ Masters: [ rhel7-4 ] -+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ] -+ -diff --git a/cts/scheduler/on_fail_demote3.xml b/cts/scheduler/on_fail_demote3.xml -new file mode 100644 -index 0000000..a7b6806 ---- /dev/null -+++ b/cts/scheduler/on_fail_demote3.xml -@@ -0,0 +1,221 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/on_fail_demote4.dot b/cts/scheduler/on_fail_demote4.dot -new file mode 100644 -index 0000000..4715cd3 ---- /dev/null -+++ b/cts/scheduler/on_fail_demote4.dot -@@ -0,0 +1,383 @@ -+ digraph "g" { -+"Cancel rsc1_monitor_11000 rhel7-3" -> "rsc1_promote_0 rhel7-3" [ style = bold] -+"Cancel rsc1_monitor_11000 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"Cancel rsc2_monitor_11000 rhel7-3" -> "rsc2_promote_0 rhel7-3" [ style = bold] -+"Cancel rsc2_monitor_11000 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"Fencing_monitor_120000 rhel7-5" [ style=bold color="green" fontcolor="black"] -+"Fencing_start_0 rhel7-5" -> "Fencing_monitor_120000 rhel7-5" [ style = bold] -+"Fencing_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"] -+"Fencing_stop_0 rhel7-4" -> "Fencing_start_0 rhel7-5" [ style = bold] -+"Fencing_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"] -+"bundled_demote_0 stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"bundled_demote_0 stateful-bundle-0" -> "bundled_stop_0 stateful-bundle-0" [ style = bold] -+"bundled_demote_0 stateful-bundle-0" -> "stateful-bundle-master_demoted_0" [ style = bold] -+"bundled_demote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="orange"] -+"bundled_monitor_10000 stateful-bundle-0" [ style=bold color="green" fontcolor="black"] -+"bundled_monitor_11000 stateful-bundle-2" [ style=bold color="green" fontcolor="black"] -+"bundled_promote_0 stateful-bundle-0" -> "bundled_monitor_10000 stateful-bundle-0" [ style = bold] -+"bundled_promote_0 stateful-bundle-0" -> "stateful-bundle-master_promoted_0" [ style = bold] -+"bundled_promote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"] -+"bundled_start_0 stateful-bundle-0" -> "bundled_monitor_10000 stateful-bundle-0" [ style = bold] -+"bundled_start_0 stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"bundled_start_0 stateful-bundle-0" -> "bundled_start_0 stateful-bundle-2" [ style = bold] -+"bundled_start_0 stateful-bundle-0" -> "stateful-bundle-master_running_0" [ style = bold] -+"bundled_start_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"] -+"bundled_start_0 stateful-bundle-2" -> "bundled_monitor_11000 stateful-bundle-2" [ style = bold] -+"bundled_start_0 stateful-bundle-2" -> "stateful-bundle-master_running_0" [ style = bold] -+"bundled_start_0 stateful-bundle-2" [ style=bold color="green" fontcolor="black"] -+"bundled_stop_0 stateful-bundle-0" -> "bundled_start_0 stateful-bundle-0" [ style = bold] -+"bundled_stop_0 stateful-bundle-0" -> "stateful-bundle-master_stopped_0" [ style = bold] -+"bundled_stop_0 stateful-bundle-0" [ style=bold color="green" fontcolor="orange"] -+"bundled_stop_0 stateful-bundle-2" -> "bundled_start_0 stateful-bundle-2" [ style = bold] -+"bundled_stop_0 stateful-bundle-2" -> "bundled_stop_0 stateful-bundle-0" [ style = bold] -+"bundled_stop_0 stateful-bundle-2" -> "stateful-bundle-master_stopped_0" [ style = bold] -+"bundled_stop_0 stateful-bundle-2" [ style=bold color="green" fontcolor="orange"] -+"container2_monitor_20000 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"container2_start_0 rhel7-3" -> "container2_monitor_20000 rhel7-3" [ style = bold] -+"container2_start_0 rhel7-3" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"container2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold] -+"container2_start_0 rhel7-3" -> "lxc2_start_0 rhel7-3" [ style = bold] -+"container2_start_0 rhel7-3" -> "rsc1_start_0 lxc2" [ style = bold] -+"container2_start_0 rhel7-3" -> "rsc2_start_0 lxc2" [ style = bold] -+"container2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"container2_stop_0 rhel7-3" -> "container2_start_0 rhel7-3" [ style = bold] -+"container2_stop_0 rhel7-3" -> "stonith 'reboot' lxc2" [ style = bold] -+"container2_stop_0 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold] -+"lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc2" [ style = bold] -+"lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold] -+"lxc-ms-master_demoted_0" -> "lxc-ms-master_start_0" [ style = bold] -+"lxc-ms-master_demoted_0" -> "lxc-ms-master_stop_0" [ style = bold] -+"lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms-master_running_0" -> "lxc-ms-master_promote_0" [ style = bold] -+"lxc-ms-master_running_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms-master_start_0" -> "lxc-ms-master_running_0" [ style = bold] -+"lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc2" [ style = bold] -+"lxc-ms-master_start_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms-master_stop_0" -> "lxc-ms-master_stopped_0" [ style = bold] -+"lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc2" [ style = bold] -+"lxc-ms-master_stop_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms-master_stopped_0" -> "lxc-ms-master_promote_0" [ style = bold] -+"lxc-ms-master_stopped_0" -> "lxc-ms-master_start_0" [ style = bold] -+"lxc-ms-master_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms_demote_0 lxc2" -> "lxc-ms-master_demoted_0" [ style = bold] -+"lxc-ms_demote_0 lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"lxc-ms_demote_0 lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold] -+"lxc-ms_demote_0 lxc2" [ style=bold color="green" fontcolor="orange"] -+"lxc-ms_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] -+"lxc-ms_promote_0 lxc2" -> "lxc-ms-master_promoted_0" [ style = bold] -+"lxc-ms_promote_0 lxc2" -> "lxc-ms_monitor_10000 lxc2" [ style = bold] -+"lxc-ms_promote_0 lxc2" [ style=bold color="green" fontcolor="black"] -+"lxc-ms_start_0 lxc2" -> "lxc-ms-master_running_0" [ style = bold] -+"lxc-ms_start_0 lxc2" -> "lxc-ms_monitor_10000 lxc2" [ style = bold] -+"lxc-ms_start_0 lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"lxc-ms_start_0 lxc2" [ style=bold color="green" fontcolor="black"] -+"lxc-ms_stop_0 lxc2" -> "lxc-ms-master_stopped_0" [ style = bold] -+"lxc-ms_stop_0 lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold] -+"lxc-ms_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"] -+"lxc2_monitor_30000 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"lxc2_start_0 rhel7-3" -> "lxc-ms_monitor_10000 lxc2" [ style = bold] -+"lxc2_start_0 rhel7-3" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"lxc2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold] -+"lxc2_start_0 rhel7-3" -> "lxc2_monitor_30000 rhel7-3" [ style = bold] -+"lxc2_start_0 rhel7-3" -> "rsc1_monitor_11000 lxc2" [ style = bold] -+"lxc2_start_0 rhel7-3" -> "rsc1_start_0 lxc2" [ style = bold] -+"lxc2_start_0 rhel7-3" -> "rsc2_monitor_11000 lxc2" [ style = bold] -+"lxc2_start_0 rhel7-3" -> "rsc2_start_0 lxc2" [ style = bold] -+"lxc2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"lxc2_stop_0 rhel7-3" -> "container2_stop_0 rhel7-3" [ style = bold] -+"lxc2_stop_0 rhel7-3" -> "lxc2_start_0 rhel7-3" [ style = bold] -+"lxc2_stop_0 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"remote-rhel7-2_monitor_60000 rhel7-1" [ style=bold color="green" fontcolor="black"] -+"remote-rhel7-2_start_0 rhel7-1" -> "remote-rhel7-2_monitor_60000 rhel7-1" [ style = bold] -+"remote-rhel7-2_start_0 rhel7-1" [ style=bold color="green" fontcolor="black"] -+"remote-rhel7-2_stop_0 rhel7-1" -> "remote-rhel7-2_start_0 rhel7-1" [ style = bold] -+"remote-rhel7-2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="black"] -+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold] -+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold] -+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_demoted_0" -> "rsc1-clone_promote_0" [ style = bold] -+"rsc1-clone_demoted_0" -> "rsc1-clone_start_0" [ style = bold] -+"rsc1-clone_demoted_0" -> "rsc1-clone_stop_0" [ style = bold] -+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_promote_0" -> "rsc1_promote_0 rhel7-3" [ style = bold] -+"rsc1-clone_promote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_running_0" -> "rsc1-clone_promote_0" [ style = bold] -+"rsc1-clone_running_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_start_0" -> "rsc1-clone_running_0" [ style = bold] -+"rsc1-clone_start_0" -> "rsc1_start_0 lxc2" [ style = bold] -+"rsc1-clone_start_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_stop_0" -> "rsc1-clone_stopped_0" [ style = bold] -+"rsc1-clone_stop_0" -> "rsc1_stop_0 lxc2" [ style = bold] -+"rsc1-clone_stop_0" -> "rsc1_stop_0 remote-rhel7-2" [ style = bold] -+"rsc1-clone_stop_0" -> "rsc1_stop_0 rhel7-4" [ style = bold] -+"rsc1-clone_stop_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_stopped_0" -> "rsc1-clone_promote_0" [ style = bold] -+"rsc1-clone_stopped_0" -> "rsc1-clone_start_0" [ style = bold] -+"rsc1-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold] -+"rsc1_demote_0 rhel7-4" -> "rsc1_stop_0 rhel7-4" [ style = bold] -+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="orange"] -+"rsc1_monitor_10000 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"rsc1_monitor_11000 lxc2" [ style=bold color="green" fontcolor="black"] -+"rsc1_promote_0 rhel7-3" -> "rsc1-clone_promoted_0" [ style = bold] -+"rsc1_promote_0 rhel7-3" -> "rsc1_monitor_10000 rhel7-3" [ style = bold] -+"rsc1_promote_0 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"rsc1_start_0 lxc2" -> "rsc1-clone_running_0" [ style = bold] -+"rsc1_start_0 lxc2" -> "rsc1_monitor_11000 lxc2" [ style = bold] -+"rsc1_start_0 lxc2" [ style=bold color="green" fontcolor="black"] -+"rsc1_stop_0 lxc2" -> "rsc1-clone_stopped_0" [ style = bold] -+"rsc1_stop_0 lxc2" -> "rsc1_start_0 lxc2" [ style = bold] -+"rsc1_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"] -+"rsc1_stop_0 remote-rhel7-2" -> "remote-rhel7-2_stop_0 rhel7-1" [ style = bold] -+"rsc1_stop_0 remote-rhel7-2" -> "rsc1-clone_stopped_0" [ style = bold] -+"rsc1_stop_0 remote-rhel7-2" [ style=bold color="green" fontcolor="orange"] -+"rsc1_stop_0 rhel7-4" -> "rsc1-clone_stopped_0" [ style = bold] -+"rsc1_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"] -+"rsc2-master_demote_0" -> "rsc2-master_demoted_0" [ style = bold] -+"rsc2-master_demote_0" -> "rsc2_demote_0 remote-rhel7-2" [ style = bold] -+"rsc2-master_demote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2-master_demoted_0" -> "rsc2-master_promote_0" [ style = bold] -+"rsc2-master_demoted_0" -> "rsc2-master_start_0" [ style = bold] -+"rsc2-master_demoted_0" -> "rsc2-master_stop_0" [ style = bold] -+"rsc2-master_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2-master_promote_0" -> "rsc2_promote_0 rhel7-3" [ style = bold] -+"rsc2-master_promote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2-master_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2-master_running_0" -> "rsc2-master_promote_0" [ style = bold] -+"rsc2-master_running_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2-master_start_0" -> "rsc2-master_running_0" [ style = bold] -+"rsc2-master_start_0" -> "rsc2_start_0 lxc2" [ style = bold] -+"rsc2-master_start_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2-master_stop_0" -> "rsc2-master_stopped_0" [ style = bold] -+"rsc2-master_stop_0" -> "rsc2_stop_0 lxc2" [ style = bold] -+"rsc2-master_stop_0" -> "rsc2_stop_0 remote-rhel7-2" [ style = bold] -+"rsc2-master_stop_0" -> "rsc2_stop_0 rhel7-4" [ style = bold] -+"rsc2-master_stop_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2-master_stopped_0" -> "rsc2-master_promote_0" [ style = bold] -+"rsc2-master_stopped_0" -> "rsc2-master_start_0" [ style = bold] -+"rsc2-master_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"rsc2_demote_0 remote-rhel7-2" -> "rsc2-master_demoted_0" [ style = bold] -+"rsc2_demote_0 remote-rhel7-2" -> "rsc2_stop_0 remote-rhel7-2" [ style = bold] -+"rsc2_demote_0 remote-rhel7-2" [ style=bold color="green" fontcolor="orange"] -+"rsc2_monitor_10000 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"rsc2_monitor_11000 lxc2" [ style=bold color="green" fontcolor="black"] -+"rsc2_promote_0 rhel7-3" -> "rsc2-master_promoted_0" [ style = bold] -+"rsc2_promote_0 rhel7-3" -> "rsc2_monitor_10000 rhel7-3" [ style = bold] -+"rsc2_promote_0 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"rsc2_start_0 lxc2" -> "rsc2-master_running_0" [ style = bold] -+"rsc2_start_0 lxc2" -> "rsc2_monitor_11000 lxc2" [ style = bold] -+"rsc2_start_0 lxc2" [ style=bold color="green" fontcolor="black"] -+"rsc2_stop_0 lxc2" -> "rsc2-master_stopped_0" [ style = bold] -+"rsc2_stop_0 lxc2" -> "rsc2_start_0 lxc2" [ style = bold] -+"rsc2_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"] -+"rsc2_stop_0 remote-rhel7-2" -> "remote-rhel7-2_stop_0 rhel7-1" [ style = bold] -+"rsc2_stop_0 remote-rhel7-2" -> "rsc2-master_stopped_0" [ style = bold] -+"rsc2_stop_0 remote-rhel7-2" [ style=bold color="green" fontcolor="orange"] -+"rsc2_stop_0 rhel7-4" -> "rsc2-master_stopped_0" [ style = bold] -+"rsc2_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-0_monitor_30000 rhel7-5" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-0_start_0 rhel7-5" -> "bundled_monitor_10000 stateful-bundle-0" [ style = bold] -+"stateful-bundle-0_start_0 rhel7-5" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"stateful-bundle-0_start_0 rhel7-5" -> "bundled_start_0 stateful-bundle-0" [ style = bold] -+"stateful-bundle-0_start_0 rhel7-5" -> "stateful-bundle-0_monitor_30000 rhel7-5" [ style = bold] -+"stateful-bundle-0_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-0_stop_0 rhel7-5" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold] -+"stateful-bundle-0_stop_0 rhel7-5" -> "stateful-bundle-docker-0_stop_0 rhel7-5" [ style = bold] -+"stateful-bundle-0_stop_0 rhel7-5" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-2_monitor_30000 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-2_start_0 rhel7-3" -> "bundled_monitor_11000 stateful-bundle-2" [ style = bold] -+"stateful-bundle-2_start_0 rhel7-3" -> "bundled_start_0 stateful-bundle-2" [ style = bold] -+"stateful-bundle-2_start_0 rhel7-3" -> "stateful-bundle-2_monitor_30000 rhel7-3" [ style = bold] -+"stateful-bundle-2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-2_stop_0 rhel7-4" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold] -+"stateful-bundle-2_stop_0 rhel7-4" -> "stateful-bundle-docker-2_stop_0 rhel7-4" [ style = bold] -+"stateful-bundle-2_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-docker-0_monitor_60000 rhel7-5" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-docker-0_start_0 rhel7-5" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"stateful-bundle-docker-0_start_0 rhel7-5" -> "bundled_start_0 stateful-bundle-0" [ style = bold] -+"stateful-bundle-docker-0_start_0 rhel7-5" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold] -+"stateful-bundle-docker-0_start_0 rhel7-5" -> "stateful-bundle-docker-0_monitor_60000 rhel7-5" [ style = bold] -+"stateful-bundle-docker-0_start_0 rhel7-5" -> "stateful-bundle_running_0" [ style = bold] -+"stateful-bundle-docker-0_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-docker-0_stop_0 rhel7-5" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold] -+"stateful-bundle-docker-0_stop_0 rhel7-5" -> "stateful-bundle_stopped_0" [ style = bold] -+"stateful-bundle-docker-0_stop_0 rhel7-5" -> "stonith 'reboot' stateful-bundle-0" [ style = bold] -+"stateful-bundle-docker-0_stop_0 rhel7-5" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-docker-2_monitor_60000 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-docker-2_start_0 rhel7-3" -> "bundled_start_0 stateful-bundle-2" [ style = bold] -+"stateful-bundle-docker-2_start_0 rhel7-3" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold] -+"stateful-bundle-docker-2_start_0 rhel7-3" -> "stateful-bundle-docker-2_monitor_60000 rhel7-3" [ style = bold] -+"stateful-bundle-docker-2_start_0 rhel7-3" -> "stateful-bundle_running_0" [ style = bold] -+"stateful-bundle-docker-2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-docker-2_stop_0 rhel7-4" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold] -+"stateful-bundle-docker-2_stop_0 rhel7-4" -> "stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" [ style = bold] -+"stateful-bundle-docker-2_stop_0 rhel7-4" -> "stateful-bundle_stopped_0" [ style = bold] -+"stateful-bundle-docker-2_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-ip-192.168.122.133_monitor_60000 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold] -+"stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" -> "stateful-bundle-ip-192.168.122.133_monitor_60000 rhel7-3" [ style = bold] -+"stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] -+"stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold] -+"stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-master_demote_0" -> "bundled_demote_0 stateful-bundle-0" [ style = bold] -+"stateful-bundle-master_demote_0" -> "stateful-bundle-master_demoted_0" [ style = bold] -+"stateful-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_promote_0" [ style = bold] -+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_start_0" [ style = bold] -+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_stop_0" [ style = bold] -+"stateful-bundle-master_demoted_0" -> "stateful-bundle_demoted_0" [ style = bold] -+"stateful-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-master_promote_0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"stateful-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-master_promoted_0" -> "stateful-bundle_promoted_0" [ style = bold] -+"stateful-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-master_running_0" -> "stateful-bundle-master_promote_0" [ style = bold] -+"stateful-bundle-master_running_0" -> "stateful-bundle_running_0" [ style = bold] -+"stateful-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-master_start_0" -> "bundled_start_0 stateful-bundle-0" [ style = bold] -+"stateful-bundle-master_start_0" -> "bundled_start_0 stateful-bundle-2" [ style = bold] -+"stateful-bundle-master_start_0" -> "stateful-bundle-master_running_0" [ style = bold] -+"stateful-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-master_stop_0" -> "bundled_stop_0 stateful-bundle-0" [ style = bold] -+"stateful-bundle-master_stop_0" -> "bundled_stop_0 stateful-bundle-2" [ style = bold] -+"stateful-bundle-master_stop_0" -> "stateful-bundle-master_stopped_0" [ style = bold] -+"stateful-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle-master_stopped_0" -> "stateful-bundle-master_promote_0" [ style = bold] -+"stateful-bundle-master_stopped_0" -> "stateful-bundle-master_start_0" [ style = bold] -+"stateful-bundle-master_stopped_0" -> "stateful-bundle_stopped_0" [ style = bold] -+"stateful-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_demote_0" -> "stateful-bundle-master_demote_0" [ style = bold] -+"stateful-bundle_demote_0" -> "stateful-bundle_demoted_0" [ style = bold] -+"stateful-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_demoted_0" -> "stateful-bundle_promote_0" [ style = bold] -+"stateful-bundle_demoted_0" -> "stateful-bundle_start_0" [ style = bold] -+"stateful-bundle_demoted_0" -> "stateful-bundle_stop_0" [ style = bold] -+"stateful-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_promote_0" -> "stateful-bundle-master_promote_0" [ style = bold] -+"stateful-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_running_0" -> "stateful-bundle_promote_0" [ style = bold] -+"stateful-bundle_running_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_start_0" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold] -+"stateful-bundle_start_0" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold] -+"stateful-bundle_start_0" -> "stateful-bundle-master_start_0" [ style = bold] -+"stateful-bundle_start_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_stop_0" -> "bundled_stop_0 stateful-bundle-0" [ style = bold] -+"stateful-bundle_stop_0" -> "bundled_stop_0 stateful-bundle-2" [ style = bold] -+"stateful-bundle_stop_0" -> "stateful-bundle-docker-0_stop_0 rhel7-5" [ style = bold] -+"stateful-bundle_stop_0" -> "stateful-bundle-docker-2_stop_0 rhel7-4" [ style = bold] -+"stateful-bundle_stop_0" -> "stateful-bundle-master_stop_0" [ style = bold] -+"stateful-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] -+"stateful-bundle_stopped_0" -> "stateful-bundle_promote_0" [ style = bold] -+"stateful-bundle_stopped_0" -> "stateful-bundle_start_0" [ style = bold] -+"stateful-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] -+"stonith 'reboot' lxc2" -> "Fencing_start_0 rhel7-5" [ style = bold] -+"stonith 'reboot' lxc2" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"stonith 'reboot' lxc2" -> "bundled_start_0 stateful-bundle-0" [ style = bold] -+"stonith 'reboot' lxc2" -> "bundled_start_0 stateful-bundle-2" [ style = bold] -+"stonith 'reboot' lxc2" -> "container2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' lxc2" -> "lxc-ms-master_stop_0" [ style = bold] -+"stonith 'reboot' lxc2" -> "lxc-ms_demote_0 lxc2" [ style = bold] -+"stonith 'reboot' lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"stonith 'reboot' lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold] -+"stonith 'reboot' lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold] -+"stonith 'reboot' lxc2" -> "lxc2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' lxc2" -> "remote-rhel7-2_start_0 rhel7-1" [ style = bold] -+"stonith 'reboot' lxc2" -> "rsc1-clone_stop_0" [ style = bold] -+"stonith 'reboot' lxc2" -> "rsc1_promote_0 rhel7-3" [ style = bold] -+"stonith 'reboot' lxc2" -> "rsc1_start_0 lxc2" [ style = bold] -+"stonith 'reboot' lxc2" -> "rsc1_stop_0 lxc2" [ style = bold] -+"stonith 'reboot' lxc2" -> "rsc2-master_stop_0" [ style = bold] -+"stonith 'reboot' lxc2" -> "rsc2_promote_0 rhel7-3" [ style = bold] -+"stonith 'reboot' lxc2" -> "rsc2_start_0 lxc2" [ style = bold] -+"stonith 'reboot' lxc2" -> "rsc2_stop_0 lxc2" [ style = bold] -+"stonith 'reboot' lxc2" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold] -+"stonith 'reboot' lxc2" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' lxc2" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold] -+"stonith 'reboot' lxc2" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' lxc2" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' lxc2" [ style=bold color="green" fontcolor="orange"] -+"stonith 'reboot' remote-rhel7-2" -> "Fencing_start_0 rhel7-5" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "bundled_start_0 stateful-bundle-0" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "bundled_start_0 stateful-bundle-2" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "container2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "lxc-ms_start_0 lxc2" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "lxc2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "remote-rhel7-2_start_0 rhel7-1" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "rsc1-clone_stop_0" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "rsc1_promote_0 rhel7-3" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "rsc1_start_0 lxc2" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "rsc1_stop_0 remote-rhel7-2" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "rsc2-master_stop_0" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "rsc2_demote_0 remote-rhel7-2" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "rsc2_promote_0 rhel7-3" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "rsc2_start_0 lxc2" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "rsc2_stop_0 remote-rhel7-2" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" -> "stonith 'reboot' rhel7-4" [ style = bold] -+"stonith 'reboot' remote-rhel7-2" [ style=bold color="green" fontcolor="black"] -+"stonith 'reboot' rhel7-4" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "bundled_start_0 stateful-bundle-0" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "bundled_start_0 stateful-bundle-2" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "container2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "lxc-ms_start_0 lxc2" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "rsc1-clone_stop_0" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "rsc1_demote_0 rhel7-4" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "rsc1_promote_0 rhel7-3" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "rsc1_start_0 lxc2" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "rsc1_stop_0 rhel7-4" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "rsc2-master_stop_0" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "rsc2_promote_0 rhel7-3" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "rsc2_start_0 lxc2" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "rsc2_stop_0 rhel7-4" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "stateful-bundle-docker-2_stop_0 rhel7-4" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" [ style = bold] -+"stonith 'reboot' rhel7-4" -> "stonith 'reboot' stateful-bundle-2" [ style = bold] -+"stonith 'reboot' rhel7-4" [ style=bold color="green" fontcolor="black"] -+"stonith 'reboot' stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "bundled_start_0 stateful-bundle-0" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "bundled_start_0 stateful-bundle-2" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "container2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "lxc-ms_start_0 lxc2" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "rsc1_promote_0 rhel7-3" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "rsc1_start_0 lxc2" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "rsc2_promote_0 rhel7-3" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "rsc2_start_0 lxc2" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-master_stop_0" [ style = bold] -+"stonith 'reboot' stateful-bundle-0" [ style=bold color="green" fontcolor="orange"] -+"stonith 'reboot' stateful-bundle-2" -> "bundled_promote_0 stateful-bundle-0" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "bundled_start_0 stateful-bundle-0" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "bundled_start_0 stateful-bundle-2" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "container2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "lxc-ms_promote_0 lxc2" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "lxc-ms_start_0 lxc2" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "rsc1_promote_0 rhel7-3" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "rsc1_start_0 lxc2" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "rsc2_promote_0 rhel7-3" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "rsc2_start_0 lxc2" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-master_stop_0" [ style = bold] -+"stonith 'reboot' stateful-bundle-2" [ style=bold color="green" fontcolor="orange"] -+} -diff --git a/cts/scheduler/on_fail_demote4.exp b/cts/scheduler/on_fail_demote4.exp -new file mode 100644 -index 0000000..0789a12 ---- /dev/null -+++ b/cts/scheduler/on_fail_demote4.exp -@@ -0,0 +1,1818 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/on_fail_demote4.scores b/cts/scheduler/on_fail_demote4.scores -new file mode 100644 -index 0000000..cde3fec ---- /dev/null -+++ b/cts/scheduler/on_fail_demote4.scores -@@ -0,0 +1,470 @@ -+Allocation scores: -+Using the original execution date of: 2020-06-16 19:23:21Z -+bundled:0 promotion score on stateful-bundle-0: 10 -+bundled:1 promotion score on stateful-bundle-1: 5 -+bundled:2 promotion score on stateful-bundle-2: 5 -+lxc-ms:0 promotion score on lxc2: INFINITY -+lxc-ms:1 promotion score on lxc1: INFINITY -+pcmk__bundle_allocate: bundled:0 allocation score on stateful-bundle-0: 501 -+pcmk__bundle_allocate: bundled:1 allocation score on stateful-bundle-1: 501 -+pcmk__bundle_allocate: bundled:2 allocation score on stateful-bundle-2: 501 -+pcmk__bundle_allocate: stateful-bundle allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc1: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc2: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on remote-rhel7-2: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-1: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-3: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-4: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-5: 0 -+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-0: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-1: -INFINITY -+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-2: -INFINITY -+pcmk__clone_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY -+pcmk__clone_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY -+pcmk__clone_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY -+pcmk__clone_allocate: lxc-ms-master allocation score on lxc1: INFINITY -+pcmk__clone_allocate: lxc-ms-master allocation score on lxc2: INFINITY -+pcmk__clone_allocate: lxc-ms-master allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-1: 0 -+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-3: 0 -+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-4: 0 -+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-5: 0 -+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc1: INFINITY -+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc2: INFINITY -+pcmk__clone_allocate: lxc-ms:0 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc1: INFINITY -+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc2: INFINITY -+pcmk__clone_allocate: lxc-ms:1 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 1 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6 -+pcmk__clone_allocate: rsc1:3 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on remote-rhel7-2: 1 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on lxc2: 6 -+pcmk__clone_allocate: rsc1:5 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on lxc1: 6 -+pcmk__clone_allocate: rsc1:6 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2-master allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2-master allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2-master allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 1 -+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6 -+pcmk__clone_allocate: rsc2:3 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on remote-rhel7-2: 1 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on lxc1: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on lxc2: 6 -+pcmk__clone_allocate: rsc2:5 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on lxc1: 6 -+pcmk__clone_allocate: rsc2:6 allocation score on lxc2: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on remote-rhel7-2: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc1: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc2: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on remote-rhel7-2: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-1: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-3: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-4: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-5: -INFINITY -+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-0: 0 -+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-1: 0 -+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-2: 0 -+pcmk__native_allocate: Fencing allocation score on lxc1: -INFINITY -+pcmk__native_allocate: Fencing allocation score on lxc2: -INFINITY -+pcmk__native_allocate: Fencing allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0 -+pcmk__native_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY -+pcmk__native_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY -+pcmk__native_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY -+pcmk__native_allocate: container1 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: container1 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: container1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: container1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: container1 allocation score on rhel7-3: INFINITY -+pcmk__native_allocate: container1 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: container1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: container2 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: container2 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: container2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: container2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: container2 allocation score on rhel7-3: INFINITY -+pcmk__native_allocate: container2 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: container2 allocation score on rhel7-5: 0 -+pcmk__native_allocate: lxc-ms:0 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: lxc-ms:0 allocation score on lxc2: INFINITY -+pcmk__native_allocate: lxc-ms:0 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-1: 0 -+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-3: 0 -+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-5: 0 -+pcmk__native_allocate: lxc-ms:1 allocation score on lxc1: INFINITY -+pcmk__native_allocate: lxc-ms:1 allocation score on lxc2: INFINITY -+pcmk__native_allocate: lxc-ms:1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-3: 0 -+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: lxc1 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: lxc1 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: lxc1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: lxc1 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: lxc1 allocation score on rhel7-3: 0 -+pcmk__native_allocate: lxc1 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: lxc1 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on rhel7-3: 0 -+pcmk__native_allocate: lxc2 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: lxc2 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: remote-rhel7-2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-3: 0 -+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-4: 0 -+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc1:0 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:1 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc1:1 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc1:1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc1:2 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc1:2 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc1:2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6 -+pcmk__native_allocate: rsc1:3 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc1:3 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc1:3 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6 -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:5 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: rsc1:5 allocation score on lxc2: 6 -+pcmk__native_allocate: rsc1:5 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:5 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:5 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:5 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:5 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:6 allocation score on lxc1: 6 -+pcmk__native_allocate: rsc1:6 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc1:6 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:6 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:6 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:6 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:6 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:0 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: rsc2:0 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: rsc2:0 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:1 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc2:1 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc2:1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6 -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc2:2 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc2:2 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc2:2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6 -+pcmk__native_allocate: rsc2:3 allocation score on lxc1: 0 -+pcmk__native_allocate: rsc2:3 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc2:3 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6 -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:5 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: rsc2:5 allocation score on lxc2: 6 -+pcmk__native_allocate: rsc2:5 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc2:5 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc2:5 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:5 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:5 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2:6 allocation score on lxc1: 6 -+pcmk__native_allocate: rsc2:6 allocation score on lxc2: 0 -+pcmk__native_allocate: rsc2:6 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: rsc2:6 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc2:6 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc2:6 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc2:6 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-1: 0 -+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-4: 0 -+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-5: 10000 -+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-1: 10000 -+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-4: 0 -+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-3: 10000 -+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-4: 0 -+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-5: 0 -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc1: -10000 -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc2: -10000 -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc1: -10000 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc2: -10000 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: 0 -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc1: -10000 -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc2: -10000 -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0 -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0 -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: 0 -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: -INFINITY -+rsc1:0 promotion score on none: 0 -+rsc1:1 promotion score on rhel7-3: 5 -+rsc1:2 promotion score on rhel7-5: 5 -+rsc1:3 promotion score on rhel7-1: 5 -+rsc1:4 promotion score on none: 0 -+rsc1:5 promotion score on lxc2: 5 -+rsc1:6 promotion score on lxc1: 5 -+rsc2:0 promotion score on none: 0 -+rsc2:1 promotion score on rhel7-3: 5 -+rsc2:2 promotion score on rhel7-5: 5 -+rsc2:3 promotion score on rhel7-1: 5 -+rsc2:4 promotion score on none: 0 -+rsc2:5 promotion score on lxc2: 5 -+rsc2:6 promotion score on lxc1: 5 -diff --git a/cts/scheduler/on_fail_demote4.summary b/cts/scheduler/on_fail_demote4.summary -new file mode 100644 -index 0000000..20520ff ---- /dev/null -+++ b/cts/scheduler/on_fail_demote4.summary -@@ -0,0 +1,187 @@ -+Using the original execution date of: 2020-06-16 19:23:21Z -+ -+Current cluster status: -+RemoteNode remote-rhel7-2: UNCLEAN (offline) -+Node rhel7-4 (4): UNCLEAN (offline) -+Online: [ rhel7-1 rhel7-3 rhel7-5 ] -+GuestOnline: [ lxc1:container1 stateful-bundle-1:stateful-bundle-docker-1 ] -+ -+ Fencing (stonith:fence_xvm): Started rhel7-4 (UNCLEAN) -+ Clone Set: rsc1-clone [rsc1] (promotable) -+ rsc1 (ocf::pacemaker:Stateful): Master rhel7-4 (UNCLEAN) -+ rsc1 (ocf::pacemaker:Stateful): Slave remote-rhel7-2 (UNCLEAN) -+ Slaves: [ lxc1 rhel7-1 rhel7-3 rhel7-5 ] -+ Clone Set: rsc2-master [rsc2] (promotable) -+ rsc2 (ocf::pacemaker:Stateful): Slave rhel7-4 (UNCLEAN) -+ rsc2 (ocf::pacemaker:Stateful): Master remote-rhel7-2 (UNCLEAN) -+ Slaves: [ lxc1 rhel7-1 rhel7-3 rhel7-5 ] -+ remote-rhel7-2 (ocf::pacemaker:remote): FAILED rhel7-1 -+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3 -+ container2 (ocf::heartbeat:VirtualDomain): FAILED rhel7-3 -+ Clone Set: lxc-ms-master [lxc-ms] (promotable) -+ Slaves: [ lxc1 ] -+ Stopped: [ remote-rhel7-2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ] -+ Container bundle set: stateful-bundle [pcmktest:http] -+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): FAILED Master rhel7-5 -+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1 -+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): FAILED rhel7-4 (UNCLEAN) -+ -+Transition Summary: -+ * Fence (reboot) stateful-bundle-2 (resource: stateful-bundle-docker-2) 'guest is unclean' -+ * Fence (reboot) stateful-bundle-0 (resource: stateful-bundle-docker-0) 'guest is unclean' -+ * Fence (reboot) lxc2 (resource: container2) 'guest is unclean' -+ * Fence (reboot) remote-rhel7-2 'remote connection is unrecoverable' -+ * Fence (reboot) rhel7-4 'peer is no longer part of the cluster' -+ * Move Fencing ( rhel7-4 -> rhel7-5 ) -+ * Stop rsc1:0 ( Master rhel7-4 ) due to node availability -+ * Promote rsc1:1 ( Slave -> Master rhel7-3 ) -+ * Stop rsc1:4 ( Slave remote-rhel7-2 ) due to node availability -+ * Recover rsc1:5 ( Slave lxc2 ) -+ * Stop rsc2:0 ( Slave rhel7-4 ) due to node availability -+ * Promote rsc2:1 ( Slave -> Master rhel7-3 ) -+ * Stop rsc2:4 ( Master remote-rhel7-2 ) due to node availability -+ * Recover rsc2:5 ( Slave lxc2 ) -+ * Recover remote-rhel7-2 ( rhel7-1 ) -+ * Recover container2 ( rhel7-3 ) -+ * Recover lxc-ms:0 ( Master lxc2 ) -+ * Recover stateful-bundle-docker-0 ( rhel7-5 ) -+ * Restart stateful-bundle-0 ( rhel7-5 ) due to required stateful-bundle-docker-0 start -+ * Recover bundled:0 ( Master stateful-bundle-0 ) -+ * Move stateful-bundle-ip-192.168.122.133 ( rhel7-4 -> rhel7-3 ) -+ * Recover stateful-bundle-docker-2 ( rhel7-4 -> rhel7-3 ) -+ * Move stateful-bundle-2 ( rhel7-4 -> rhel7-3 ) -+ * Recover bundled:2 ( Slave stateful-bundle-2 ) -+ * Restart lxc2 ( rhel7-3 ) due to required container2 start -+ -+Executing cluster transition: -+ * Pseudo action: Fencing_stop_0 -+ * Resource action: rsc1 cancel=11000 on rhel7-3 -+ * Pseudo action: rsc1-clone_demote_0 -+ * Resource action: rsc2 cancel=11000 on rhel7-3 -+ * Pseudo action: rsc2-master_demote_0 -+ * Pseudo action: lxc-ms-master_demote_0 -+ * Resource action: stateful-bundle-0 stop on rhel7-5 -+ * Pseudo action: stateful-bundle-2_stop_0 -+ * Resource action: lxc2 stop on rhel7-3 -+ * Pseudo action: stateful-bundle_demote_0 -+ * Fencing remote-rhel7-2 (reboot) -+ * Fencing rhel7-4 (reboot) -+ * Pseudo action: rsc1_demote_0 -+ * Pseudo action: rsc1-clone_demoted_0 -+ * Pseudo action: rsc2_demote_0 -+ * Pseudo action: rsc2-master_demoted_0 -+ * Resource action: container2 stop on rhel7-3 -+ * Pseudo action: stateful-bundle-master_demote_0 -+ * Pseudo action: stonith-stateful-bundle-2-reboot on stateful-bundle-2 -+ * Pseudo action: stonith-lxc2-reboot on lxc2 -+ * Resource action: Fencing start on rhel7-5 -+ * Pseudo action: rsc1-clone_stop_0 -+ * Pseudo action: rsc2-master_stop_0 -+ * Pseudo action: lxc-ms_demote_0 -+ * Pseudo action: lxc-ms-master_demoted_0 -+ * Pseudo action: lxc-ms-master_stop_0 -+ * Pseudo action: bundled_demote_0 -+ * Pseudo action: stateful-bundle-master_demoted_0 -+ * Pseudo action: stateful-bundle_demoted_0 -+ * Pseudo action: stateful-bundle_stop_0 -+ * Resource action: Fencing monitor=120000 on rhel7-5 -+ * Pseudo action: rsc1_stop_0 -+ * Pseudo action: rsc1_stop_0 -+ * Pseudo action: rsc1_stop_0 -+ * Pseudo action: rsc1-clone_stopped_0 -+ * Pseudo action: rsc1-clone_start_0 -+ * Pseudo action: rsc2_stop_0 -+ * Pseudo action: rsc2_stop_0 -+ * Pseudo action: rsc2_stop_0 -+ * Pseudo action: rsc2-master_stopped_0 -+ * Pseudo action: rsc2-master_start_0 -+ * Resource action: remote-rhel7-2 stop on rhel7-1 -+ * Pseudo action: lxc-ms_stop_0 -+ * Pseudo action: lxc-ms-master_stopped_0 -+ * Pseudo action: lxc-ms-master_start_0 -+ * Resource action: stateful-bundle-docker-0 stop on rhel7-5 -+ * Pseudo action: stateful-bundle-docker-2_stop_0 -+ * Pseudo action: stonith-stateful-bundle-0-reboot on stateful-bundle-0 -+ * Resource action: remote-rhel7-2 start on rhel7-1 -+ * Resource action: remote-rhel7-2 monitor=60000 on rhel7-1 -+ * Resource action: container2 start on rhel7-3 -+ * Resource action: container2 monitor=20000 on rhel7-3 -+ * Pseudo action: stateful-bundle-master_stop_0 -+ * Pseudo action: stateful-bundle-ip-192.168.122.133_stop_0 -+ * Resource action: lxc2 start on rhel7-3 -+ * Resource action: lxc2 monitor=30000 on rhel7-3 -+ * Resource action: rsc1 start on lxc2 -+ * Pseudo action: rsc1-clone_running_0 -+ * Resource action: rsc2 start on lxc2 -+ * Pseudo action: rsc2-master_running_0 -+ * Resource action: lxc-ms start on lxc2 -+ * Pseudo action: lxc-ms-master_running_0 -+ * Pseudo action: bundled_stop_0 -+ * Resource action: stateful-bundle-ip-192.168.122.133 start on rhel7-3 -+ * Resource action: rsc1 monitor=11000 on lxc2 -+ * Pseudo action: rsc1-clone_promote_0 -+ * Resource action: rsc2 monitor=11000 on lxc2 -+ * Pseudo action: rsc2-master_promote_0 -+ * Pseudo action: lxc-ms-master_promote_0 -+ * Pseudo action: bundled_stop_0 -+ * Pseudo action: stateful-bundle-master_stopped_0 -+ * Resource action: stateful-bundle-ip-192.168.122.133 monitor=60000 on rhel7-3 -+ * Pseudo action: stateful-bundle_stopped_0 -+ * Pseudo action: stateful-bundle_start_0 -+ * Resource action: rsc1 promote on rhel7-3 -+ * Pseudo action: rsc1-clone_promoted_0 -+ * Resource action: rsc2 promote on rhel7-3 -+ * Pseudo action: rsc2-master_promoted_0 -+ * Resource action: lxc-ms promote on lxc2 -+ * Pseudo action: lxc-ms-master_promoted_0 -+ * Pseudo action: stateful-bundle-master_start_0 -+ * Resource action: stateful-bundle-docker-0 start on rhel7-5 -+ * Resource action: stateful-bundle-docker-0 monitor=60000 on rhel7-5 -+ * Resource action: stateful-bundle-0 start on rhel7-5 -+ * Resource action: stateful-bundle-0 monitor=30000 on rhel7-5 -+ * Resource action: stateful-bundle-docker-2 start on rhel7-3 -+ * Resource action: stateful-bundle-2 start on rhel7-3 -+ * Resource action: rsc1 monitor=10000 on rhel7-3 -+ * Resource action: rsc2 monitor=10000 on rhel7-3 -+ * Resource action: lxc-ms monitor=10000 on lxc2 -+ * Resource action: bundled start on stateful-bundle-0 -+ * Resource action: bundled start on stateful-bundle-2 -+ * Pseudo action: stateful-bundle-master_running_0 -+ * Resource action: stateful-bundle-docker-2 monitor=60000 on rhel7-3 -+ * Resource action: stateful-bundle-2 monitor=30000 on rhel7-3 -+ * Pseudo action: stateful-bundle_running_0 -+ * Resource action: bundled monitor=11000 on stateful-bundle-2 -+ * Pseudo action: stateful-bundle_promote_0 -+ * Pseudo action: stateful-bundle-master_promote_0 -+ * Resource action: bundled promote on stateful-bundle-0 -+ * Pseudo action: stateful-bundle-master_promoted_0 -+ * Pseudo action: stateful-bundle_promoted_0 -+ * Resource action: bundled monitor=10000 on stateful-bundle-0 -+Using the original execution date of: 2020-06-16 19:23:21Z -+ -+Revised cluster status: -+Online: [ rhel7-1 rhel7-3 rhel7-5 ] -+OFFLINE: [ rhel7-4 ] -+RemoteOnline: [ remote-rhel7-2 ] -+GuestOnline: [ lxc1:container1 lxc2:container2 stateful-bundle-0:stateful-bundle-docker-0 stateful-bundle-1:stateful-bundle-docker-1 stateful-bundle-2:stateful-bundle-docker-2 ] -+ -+ Fencing (stonith:fence_xvm): Started rhel7-5 -+ Clone Set: rsc1-clone [rsc1] (promotable) -+ Masters: [ rhel7-3 ] -+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-5 ] -+ Stopped: [ remote-rhel7-2 rhel7-4 ] -+ Clone Set: rsc2-master [rsc2] (promotable) -+ Masters: [ rhel7-3 ] -+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-5 ] -+ Stopped: [ remote-rhel7-2 rhel7-4 ] -+ remote-rhel7-2 (ocf::pacemaker:remote): Started rhel7-1 -+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3 -+ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-3 -+ Clone Set: lxc-ms-master [lxc-ms] (promotable) -+ Masters: [ lxc2 ] -+ Slaves: [ lxc1 ] -+ Container bundle set: stateful-bundle [pcmktest:http] -+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): Master rhel7-5 -+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1 -+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): Slave rhel7-3 -+ -diff --git a/cts/scheduler/on_fail_demote4.xml b/cts/scheduler/on_fail_demote4.xml -new file mode 100644 -index 0000000..eb4c4cc ---- /dev/null -+++ b/cts/scheduler/on_fail_demote4.xml -@@ -0,0 +1,625 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - - -From 204961e95d9de140d998d71a0e53b5b9baa5d39e Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 26 May 2020 18:04:32 -0500 -Subject: [PATCH 12/20] Doc: Pacemaker Explained: document new on-fail="demote" - option - ---- - doc/Pacemaker_Explained/en-US/Ch-Resources.txt | 36 ++++++++++++++++++++++++++ - 1 file changed, 36 insertions(+) - -diff --git a/doc/Pacemaker_Explained/en-US/Ch-Resources.txt b/doc/Pacemaker_Explained/en-US/Ch-Resources.txt -index d8e7115..9df9243 100644 ---- a/doc/Pacemaker_Explained/en-US/Ch-Resources.txt -+++ b/doc/Pacemaker_Explained/en-US/Ch-Resources.txt -@@ -676,6 +676,10 @@ a|The action to take if this action ever fails. Allowed values: - * +ignore:+ Pretend the resource did not fail. - * +block:+ Don't perform any further operations on the resource. - * +stop:+ Stop the resource and do not start it elsewhere. -+* +demote:+ Demote the resource, without a full restart. This is valid only for -+ +promote+ actions, and for +monitor+ actions with both a nonzero +interval+ -+ and +role+ set to +Master+; for any other action, a configuration error will -+ be logged, and the default behavior will be used. - * +restart:+ Stop the resource and start it again (possibly on a different node). - * +fence:+ STONITH the node on which the resource failed. - * +standby:+ Move _all_ resources away from the node on which the resource failed. -@@ -714,6 +718,38 @@ indexterm:[Action,Property,on-fail] - - |========================================================= - -+[NOTE] -+==== -+When +on-fail+ is set to +demote+, recovery from failure by a successful demote -+causes the cluster to recalculate whether and where a new instance should be -+promoted. The node with the failure is eligible, so if master scores have not -+changed, it will be promoted again. -+ -+There is no direct equivalent of +migration-threshold+ for the master role, but -+the same effect can be achieved with a location constraint using a -+<> with a node attribute expression for the resource's fail -+count. -+ -+For example, to immediately ban the master role from a node with any failed -+promote or master monitor: -+[source,XML] -+---- -+ -+ -+ -+ -+ -+ -+---- -+ -+This example assumes that there is a promotable clone of the +my_primitive+ -+resource (note that the primitive name, not the clone name, is used in the -+rule), and that there is a recurring 10-second-interval monitor configured for -+the master role (fail count attributes specify the interval in milliseconds). -+==== -+ - [[s-resource-monitoring]] - === Monitoring Resources for Failure === - --- -1.8.3.1 - - -From d4b9117e72b178bb6f4458cd89bee13060f78dcb Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 26 May 2020 18:10:33 -0500 -Subject: [PATCH 13/20] Doc: Pacemaker Explained: correct on-fail default - ---- - doc/Pacemaker_Explained/en-US/Ch-Resources.txt | 9 +++++++-- - 1 file changed, 7 insertions(+), 2 deletions(-) - -diff --git a/doc/Pacemaker_Explained/en-US/Ch-Resources.txt b/doc/Pacemaker_Explained/en-US/Ch-Resources.txt -index 9df9243..88892db 100644 ---- a/doc/Pacemaker_Explained/en-US/Ch-Resources.txt -+++ b/doc/Pacemaker_Explained/en-US/Ch-Resources.txt -@@ -669,8 +669,13 @@ XML attributes take precedence over +nvpair+ elements if both are specified. - indexterm:[Action,Property,timeout] - - |on-fail --|restart '(except for +stop+ operations, which default to' fence 'when -- STONITH is enabled and' block 'otherwise)' -+a|Varies by action: -+ -+* +stop+: +fence+ if +stonith-enabled+ is true or +block+ otherwise -+* +demote+: +on-fail+ of the +monitor+ action with +role+ set to +Master+, if -+ present, enabled, and configured to a value other than +demote+, or +restart+ -+ otherwise -+* all other actions: +restart+ - a|The action to take if this action ever fails. Allowed values: - - * +ignore:+ Pretend the resource did not fail. --- -1.8.3.1 - - -From 0b683445318c783ecef8d6f023b35a6c056ee321 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 2 Jun 2020 15:05:56 -0500 -Subject: [PATCH 14/20] Refactor: scheduler: functionize checking quorum policy - in effect - -... for readability and ease of future changes ---- - lib/pengine/utils.c | 18 ++++++++++++++---- - 1 file changed, 14 insertions(+), 4 deletions(-) - -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index fee9efb..5d6b836 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -481,6 +481,17 @@ sort_rsc_priority(gconstpointer a, gconstpointer b) - return 0; - } - -+static enum pe_quorum_policy -+effective_quorum_policy(pe_resource_t *rsc, pe_working_set_t *data_set) -+{ -+ enum pe_quorum_policy policy = data_set->no_quorum_policy; -+ -+ if (is_set(data_set->flags, pe_flag_have_quorum)) { -+ policy = no_quorum_ignore; -+ } -+ return policy; -+} -+ - pe_action_t * - custom_action(pe_resource_t * rsc, char *key, const char *task, - pe_node_t * on_node, gboolean optional, gboolean save_action, -@@ -593,6 +604,7 @@ custom_action(pe_resource_t * rsc, char *key, const char *task, - - if (rsc != NULL) { - enum action_tasks a_task = text2task(action->task); -+ enum pe_quorum_policy quorum_policy = effective_quorum_policy(rsc, data_set); - int warn_level = LOG_TRACE; - - if (save_action) { -@@ -675,13 +687,11 @@ custom_action(pe_resource_t * rsc, char *key, const char *task, - crm_trace("Action %s requires only stonith", action->uuid); - action->runnable = TRUE; - #endif -- } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE -- && data_set->no_quorum_policy == no_quorum_stop) { -+ } else if (quorum_policy == no_quorum_stop) { - pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "no quorum", pe_action_runnable, TRUE); - crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid); - -- } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE -- && data_set->no_quorum_policy == no_quorum_freeze) { -+ } else if (quorum_policy == no_quorum_freeze) { - pe_rsc_trace(rsc, "Check resource is already active: %s %s %s %s", rsc->id, action->uuid, role2text(rsc->next_role), role2text(rsc->role)); - if (rsc->fns->active(rsc, TRUE) == FALSE || rsc->next_role > rsc->role) { - pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "quorum freeze", pe_action_runnable, TRUE); --- -1.8.3.1 - - -From b1ae359382f15e28e90d9144ca7b1d5f04820c10 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 2 Jun 2020 15:06:32 -0500 -Subject: [PATCH 15/20] Feature: scheduler: support "demote" choice for - no-quorum-policy option - -If quorum is lost, promotable resources in the master role will be demoted but -left running, and all other resources will be stopped. ---- - daemons/controld/controld_control.c | 2 +- - include/crm/pengine/pe_types.h | 3 ++- - lib/common/options.c | 1 + - lib/pengine/common.c | 2 +- - lib/pengine/pe_output.c | 14 ++++++++++++++ - lib/pengine/unpack.c | 7 +++++++ - lib/pengine/utils.c | 14 ++++++++++++++ - 7 files changed, 40 insertions(+), 3 deletions(-) - -diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c -index 7d29205..059eb7b 100644 ---- a/daemons/controld/controld_control.c -+++ b/daemons/controld/controld_control.c -@@ -626,7 +626,7 @@ static pcmk__cluster_option_t crmd_opts[] = { - - // Already documented in libpe_status (other values must be kept identical) - { -- "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, suicide", -+ "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, demote, suicide", - "stop", pcmk__valid_quorum, NULL, NULL - }, - { -diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h -index ed5eb12..f3cb4ef 100644 ---- a/include/crm/pengine/pe_types.h -+++ b/include/crm/pengine/pe_types.h -@@ -61,7 +61,8 @@ enum pe_quorum_policy { - no_quorum_freeze, - no_quorum_stop, - no_quorum_ignore, -- no_quorum_suicide -+ no_quorum_suicide, -+ no_quorum_demote - }; - - enum node_type { -diff --git a/lib/common/options.c b/lib/common/options.c -index 9399642..9e041c9 100644 ---- a/lib/common/options.c -+++ b/lib/common/options.c -@@ -407,6 +407,7 @@ pcmk__valid_quorum(const char *value) - return safe_str_eq(value, "stop") - || safe_str_eq(value, "freeze") - || safe_str_eq(value, "ignore") -+ || safe_str_eq(value, "demote") - || safe_str_eq(value, "suicide"); - } - -diff --git a/lib/pengine/common.c b/lib/pengine/common.c -index f4f2106..37f287b 100644 ---- a/lib/pengine/common.c -+++ b/lib/pengine/common.c -@@ -54,7 +54,7 @@ static pcmk__cluster_option_t pe_opts[] = { - * long description - */ - { -- "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, suicide", -+ "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, demote, suicide", - "stop", pcmk__valid_quorum, - "What to do when the cluster does not have quorum", - NULL -diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c -index 75bf0d5..ad469ab 100644 ---- a/lib/pengine/pe_output.c -+++ b/lib/pengine/pe_output.c -@@ -729,6 +729,11 @@ pe__cluster_options_html(pcmk__output_t *out, va_list args) { - out->list_item(out, NULL, "No quorum policy: Stop ALL resources"); - break; - -+ case no_quorum_demote: -+ out->list_item(out, NULL, "No quorum policy: Demote promotable " -+ "resources and stop all other resources"); -+ break; -+ - case no_quorum_ignore: - out->list_item(out, NULL, "No quorum policy: Ignore"); - break; -@@ -785,6 +790,11 @@ pe__cluster_options_text(pcmk__output_t *out, va_list args) { - out->list_item(out, NULL, "No quorum policy: Stop ALL resources"); - break; - -+ case no_quorum_demote: -+ out->list_item(out, NULL, "No quorum policy: Demote promotable " -+ "resources and stop all other resources"); -+ break; -+ - case no_quorum_ignore: - out->list_item(out, NULL, "No quorum policy: Ignore"); - break; -@@ -817,6 +827,10 @@ pe__cluster_options_xml(pcmk__output_t *out, va_list args) { - xmlSetProp(node, (pcmkXmlStr) "no-quorum-policy", (pcmkXmlStr) "stop"); - break; - -+ case no_quorum_demote: -+ xmlSetProp(node, (pcmkXmlStr) "no-quorum-policy", (pcmkXmlStr) "demote"); -+ break; -+ - case no_quorum_ignore: - xmlSetProp(node, (pcmkXmlStr) "no-quorum-policy", (pcmkXmlStr) "ignore"); - break; -diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c -index a219805..a480680 100644 ---- a/lib/pengine/unpack.c -+++ b/lib/pengine/unpack.c -@@ -268,6 +268,9 @@ unpack_config(xmlNode * config, pe_working_set_t * data_set) - } else if (safe_str_eq(value, "freeze")) { - data_set->no_quorum_policy = no_quorum_freeze; - -+ } else if (safe_str_eq(value, "demote")) { -+ data_set->no_quorum_policy = no_quorum_demote; -+ - } else if (safe_str_eq(value, "suicide")) { - if (is_set(data_set->flags, pe_flag_stonith_enabled)) { - int do_panic = 0; -@@ -297,6 +300,10 @@ unpack_config(xmlNode * config, pe_working_set_t * data_set) - case no_quorum_stop: - crm_debug("On loss of quorum: Stop ALL resources"); - break; -+ case no_quorum_demote: -+ crm_debug("On loss of quorum: " -+ "Demote promotable resources and stop other resources"); -+ break; - case no_quorum_suicide: - crm_notice("On loss of quorum: Fence all remaining nodes"); - break; -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index 5d6b836..f8b631a 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -488,6 +488,20 @@ effective_quorum_policy(pe_resource_t *rsc, pe_working_set_t *data_set) - - if (is_set(data_set->flags, pe_flag_have_quorum)) { - policy = no_quorum_ignore; -+ -+ } else if (data_set->no_quorum_policy == no_quorum_demote) { -+ switch (rsc->role) { -+ case RSC_ROLE_MASTER: -+ case RSC_ROLE_SLAVE: -+ if (rsc->next_role > RSC_ROLE_SLAVE) { -+ rsc->next_role = RSC_ROLE_SLAVE; -+ } -+ policy = no_quorum_ignore; -+ break; -+ default: -+ policy = no_quorum_stop; -+ break; -+ } - } - return policy; - } --- -1.8.3.1 - - -From 5d809e136f2927259ad570e409e3bbb68f7ce7b4 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 17 Jun 2020 12:29:50 -0500 -Subject: [PATCH 16/20] Test: scheduler: add regression test for - no-quorum-policy="demote" - ---- - cts/cts-scheduler.in | 1 + - cts/scheduler/no_quorum_demote.dot | 22 ++++ - cts/scheduler/no_quorum_demote.exp | 81 ++++++++++++ - cts/scheduler/no_quorum_demote.scores | 72 +++++++++++ - cts/scheduler/no_quorum_demote.summary | 38 ++++++ - cts/scheduler/no_quorum_demote.xml | 224 +++++++++++++++++++++++++++++++++ - 6 files changed, 438 insertions(+) - create mode 100644 cts/scheduler/no_quorum_demote.dot - create mode 100644 cts/scheduler/no_quorum_demote.exp - create mode 100644 cts/scheduler/no_quorum_demote.scores - create mode 100644 cts/scheduler/no_quorum_demote.summary - create mode 100644 cts/scheduler/no_quorum_demote.xml - -diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in -index 0e68e73..9e34379 100644 ---- a/cts/cts-scheduler.in -+++ b/cts/cts-scheduler.in -@@ -482,6 +482,7 @@ TESTS = [ - [ "on_fail_demote2", "Recovery with on-fail=\"demote\" with promotion on different node" ], - [ "on_fail_demote3", "Recovery with on-fail=\"demote\" with no promotion" ], - [ "on_fail_demote4", "Recovery with on-fail=\"demote\" on failed cluster, remote, guest, and bundle nodes" ], -+ [ "no_quorum_demote", "Promotable demotion and primitive stop with no-quorum-policy=\"demote\"" ], - ], - [ - [ "history-1", "Correctly parse stateful-1 resource state" ], -diff --git a/cts/scheduler/no_quorum_demote.dot b/cts/scheduler/no_quorum_demote.dot -new file mode 100644 -index 0000000..ea5b30c ---- /dev/null -+++ b/cts/scheduler/no_quorum_demote.dot -@@ -0,0 +1,22 @@ -+ digraph "g" { -+"Cancel rsc1_monitor_10000 rhel7-1" -> "rsc1_demote_0 rhel7-1" [ style = bold] -+"Cancel rsc1_monitor_10000 rhel7-1" [ style=bold color="green" fontcolor="black"] -+"Fencing_monitor_120000 rhel7-1" [ style=dashed color="red" fontcolor="black"] -+"Fencing_start_0 rhel7-1" -> "Fencing_monitor_120000 rhel7-1" [ style = dashed] -+"Fencing_start_0 rhel7-1" [ style=dashed color="red" fontcolor="black"] -+"Fencing_stop_0 rhel7-1" -> "Fencing_start_0 rhel7-1" [ style = dashed] -+"Fencing_stop_0 rhel7-1" [ style=bold color="green" fontcolor="black"] -+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold] -+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-1" [ style = bold] -+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"] -+"rsc1_demote_0 rhel7-1" -> "rsc1-clone_demoted_0" [ style = bold] -+"rsc1_demote_0 rhel7-1" -> "rsc1_monitor_11000 rhel7-1" [ style = bold] -+"rsc1_demote_0 rhel7-1" [ style=bold color="green" fontcolor="black"] -+"rsc1_monitor_11000 rhel7-1" [ style=bold color="green" fontcolor="black"] -+"rsc2_monitor_10000 rhel7-2" [ style=dashed color="red" fontcolor="black"] -+"rsc2_start_0 rhel7-2" -> "rsc2_monitor_10000 rhel7-2" [ style = dashed] -+"rsc2_start_0 rhel7-2" [ style=dashed color="red" fontcolor="black"] -+"rsc2_stop_0 rhel7-2" -> "rsc2_start_0 rhel7-2" [ style = dashed] -+"rsc2_stop_0 rhel7-2" [ style=bold color="green" fontcolor="black"] -+} -diff --git a/cts/scheduler/no_quorum_demote.exp b/cts/scheduler/no_quorum_demote.exp -new file mode 100644 -index 0000000..245574c ---- /dev/null -+++ b/cts/scheduler/no_quorum_demote.exp -@@ -0,0 +1,81 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -diff --git a/cts/scheduler/no_quorum_demote.scores b/cts/scheduler/no_quorum_demote.scores -new file mode 100644 -index 0000000..dddc57b ---- /dev/null -+++ b/cts/scheduler/no_quorum_demote.scores -@@ -0,0 +1,72 @@ -+Allocation scores: -+Using the original execution date of: 2020-06-17 17:26:35Z -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 11 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-2: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-2: 6 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 10 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-2: 5 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 10 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-2: 5 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 10 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-2: 5 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0 -+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-2: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0 -+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0 -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: 11 -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-2: 6 -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-2: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: -INFINITY -+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY -+pcmk__native_allocate: rsc2 allocation score on rhel7-1: 0 -+pcmk__native_allocate: rsc2 allocation score on rhel7-2: 0 -+pcmk__native_allocate: rsc2 allocation score on rhel7-3: 0 -+pcmk__native_allocate: rsc2 allocation score on rhel7-4: 0 -+pcmk__native_allocate: rsc2 allocation score on rhel7-5: 0 -+rsc1:0 promotion score on rhel7-1: 10 -+rsc1:1 promotion score on rhel7-2: 5 -+rsc1:2 promotion score on none: 0 -+rsc1:3 promotion score on none: 0 -+rsc1:4 promotion score on none: 0 -diff --git a/cts/scheduler/no_quorum_demote.summary b/cts/scheduler/no_quorum_demote.summary -new file mode 100644 -index 0000000..9b69ca1 ---- /dev/null -+++ b/cts/scheduler/no_quorum_demote.summary -@@ -0,0 +1,38 @@ -+Using the original execution date of: 2020-06-17 17:26:35Z -+ -+Current cluster status: -+Online: [ rhel7-1 rhel7-2 ] -+OFFLINE: [ rhel7-3 rhel7-4 rhel7-5 ] -+ -+ Fencing (stonith:fence_xvm): Started rhel7-1 -+ Clone Set: rsc1-clone [rsc1] (promotable) -+ Masters: [ rhel7-1 ] -+ Slaves: [ rhel7-2 ] -+ Stopped: [ rhel7-3 rhel7-4 rhel7-5 ] -+ rsc2 (ocf::pacemaker:Dummy): Started rhel7-2 -+ -+Transition Summary: -+ * Stop Fencing ( rhel7-1 ) due to no quorum -+ * Demote rsc1:0 ( Master -> Slave rhel7-1 ) -+ * Stop rsc2 ( rhel7-2 ) due to no quorum -+ -+Executing cluster transition: -+ * Resource action: Fencing stop on rhel7-1 -+ * Resource action: rsc1 cancel=10000 on rhel7-1 -+ * Pseudo action: rsc1-clone_demote_0 -+ * Resource action: rsc2 stop on rhel7-2 -+ * Resource action: rsc1 demote on rhel7-1 -+ * Pseudo action: rsc1-clone_demoted_0 -+ * Resource action: rsc1 monitor=11000 on rhel7-1 -+Using the original execution date of: 2020-06-17 17:26:35Z -+ -+Revised cluster status: -+Online: [ rhel7-1 rhel7-2 ] -+OFFLINE: [ rhel7-3 rhel7-4 rhel7-5 ] -+ -+ Fencing (stonith:fence_xvm): Stopped -+ Clone Set: rsc1-clone [rsc1] (promotable) -+ Slaves: [ rhel7-1 rhel7-2 ] -+ Stopped: [ rhel7-3 rhel7-4 rhel7-5 ] -+ rsc2 (ocf::pacemaker:Dummy): Stopped -+ -diff --git a/cts/scheduler/no_quorum_demote.xml b/cts/scheduler/no_quorum_demote.xml -new file mode 100644 -index 0000000..8497f0a ---- /dev/null -+++ b/cts/scheduler/no_quorum_demote.xml -@@ -0,0 +1,224 @@ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ --- -1.8.3.1 - - -From 015b5c012ce41a8035260522f67127135937baa2 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 28 May 2020 12:13:20 -0500 -Subject: [PATCH 17/20] Doc: Pacemaker Explained: document - no-quorum-policy=demote - ---- - doc/Pacemaker_Explained/en-US/Ch-Options.txt | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/doc/Pacemaker_Explained/en-US/Ch-Options.txt b/doc/Pacemaker_Explained/en-US/Ch-Options.txt -index faefe7c..b158f00 100644 ---- a/doc/Pacemaker_Explained/en-US/Ch-Options.txt -+++ b/doc/Pacemaker_Explained/en-US/Ch-Options.txt -@@ -181,6 +181,8 @@ What to do when the cluster does not have quorum. Allowed values: - * +ignore:+ continue all resource management - * +freeze:+ continue resource management, but don't recover resources from nodes not in the affected partition - * +stop:+ stop all resources in the affected cluster partition -+* +demote:+ demote promotable resources and stop all other resources in the -+ affected cluster partition - * +suicide:+ fence all nodes in the affected cluster partition - - | batch-limit | 0 | --- -1.8.3.1 - - -From 01c5ec67e0a6ee1395d771f8fbaf619a44ab2ca2 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 2 Jun 2020 19:23:11 -0500 -Subject: [PATCH 18/20] Low: scheduler: match initial no-quorum-policy struct - value to actual default - -It doesn't matter in practice since the actual default is parsed from the -option definition via pe_pref(), but it's confusing to have them different. ---- - lib/pengine/status.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/lib/pengine/status.c b/lib/pengine/status.c -index 8dc5095..ca34639 100644 ---- a/lib/pengine/status.c -+++ b/lib/pengine/status.c -@@ -360,7 +360,7 @@ set_working_set_defaults(pe_working_set_t * data_set) - - data_set->order_id = 1; - data_set->action_id = 1; -- data_set->no_quorum_policy = no_quorum_freeze; -+ data_set->no_quorum_policy = no_quorum_stop; - - data_set->flags = 0x0ULL; - set_bit(data_set->flags, pe_flag_stop_rsc_orphans); --- -1.8.3.1 - - -From 7eec572dbba3ade059e5206a2ba496f9da3a68bc Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 5 Jun 2020 10:02:05 -0500 -Subject: [PATCH 19/20] Build: libcrmcommon: bump CRM feature set - -... for op_expression/rsc_expression rules, on-fail=demote, and -no-quorum-policy=demote ---- - include/crm/crm.h | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/include/crm/crm.h b/include/crm/crm.h -index d2ffb61..dc2adc1 100644 ---- a/include/crm/crm.h -+++ b/include/crm/crm.h -@@ -51,7 +51,7 @@ extern "C" { - * >=3.0.13: Fail counts include operation name and interval - * >=3.2.0: DC supports PCMK_LRM_OP_INVALID and PCMK_LRM_OP_NOT_CONNECTED - */ --# define CRM_FEATURE_SET "3.3.0" -+# define CRM_FEATURE_SET "3.4.0" - - # define EOS '\0' - # define DIMOF(a) ((int) (sizeof(a)/sizeof(a[0])) ) --- -1.8.3.1 - - -From c4429d86ef00bb1749adc476f9c6874e3f5d95b9 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 16 Jun 2020 14:38:35 -0500 -Subject: [PATCH 20/20] Log: scheduler: downgrade "active on" messages to trace - -... now that they're logged more often via pcmk__rsc_is_filtered() ---- - lib/pengine/native.c | 14 +++++++------- - 1 file changed, 7 insertions(+), 7 deletions(-) - -diff --git a/lib/pengine/native.c b/lib/pengine/native.c -index f0d83d7..20658a0 100644 ---- a/lib/pengine/native.c -+++ b/lib/pengine/native.c -@@ -359,22 +359,22 @@ native_parameter(pe_resource_t * rsc, pe_node_t * node, gboolean create, const c - gboolean - native_active(pe_resource_t * rsc, gboolean all) - { -- GListPtr gIter = rsc->running_on; -- -- for (; gIter != NULL; gIter = gIter->next) { -+ for (GList *gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) { - pe_node_t *a_node = (pe_node_t *) gIter->data; - - if (a_node->details->unclean) { -- crm_debug("Resource %s: node %s is unclean", rsc->id, a_node->details->uname); -+ pe_rsc_trace(rsc, "Resource %s: node %s is unclean", -+ rsc->id, a_node->details->uname); - return TRUE; - } else if (a_node->details->online == FALSE) { -- crm_debug("Resource %s: node %s is offline", rsc->id, a_node->details->uname); -+ pe_rsc_trace(rsc, "Resource %s: node %s is offline", -+ rsc->id, a_node->details->uname); - } else { -- crm_debug("Resource %s active on %s", rsc->id, a_node->details->uname); -+ pe_rsc_trace(rsc, "Resource %s active on %s", -+ rsc->id, a_node->details->uname); - return TRUE; - } - } -- - return FALSE; - } - --- -1.8.3.1 - diff --git a/SOURCES/003-trace.patch b/SOURCES/003-trace.patch deleted file mode 100644 index e56e644..0000000 --- a/SOURCES/003-trace.patch +++ /dev/null @@ -1,30 +0,0 @@ -From 47c3e06b098c7e148c54675588d03b4d2bea40b5 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Mon, 22 Jun 2020 16:20:01 -0400 -Subject: [PATCH] Fix: libpacemaker: Don't allow a potential NULL in a format - string. - -This is only tripping up F32 s390x builds, but I don't suppose there's -any reason it couldn't come up elsewhere later. ---- - lib/pacemaker/pcmk_sched_constraints.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/lib/pacemaker/pcmk_sched_constraints.c b/lib/pacemaker/pcmk_sched_constraints.c -index 9c3a88d..d8c3e69 100644 ---- a/lib/pacemaker/pcmk_sched_constraints.c -+++ b/lib/pacemaker/pcmk_sched_constraints.c -@@ -1595,8 +1595,8 @@ custom_action_order(pe_resource_t * lh_rsc, char *lh_action_task, pe_action_t * - order = calloc(1, sizeof(pe__ordering_t)); - - crm_trace("Creating[%d] %s %s %s - %s %s %s", data_set->order_id, -- lh_rsc?lh_rsc->id:"NA", lh_action_task, lh_action?lh_action->uuid:"NA", -- rh_rsc?rh_rsc->id:"NA", rh_action_task, rh_action?rh_action->uuid:"NA"); -+ lh_rsc?lh_rsc->id:"NA", lh_action_task?lh_action_task:"NA", lh_action?lh_action->uuid:"NA", -+ rh_rsc?rh_rsc->id:"NA", rh_action_task?rh_action_task:"NA", rh_action?rh_action->uuid:"NA"); - - /* CRM_ASSERT(data_set->order_id != 291); */ - --- -1.8.3.1 - diff --git a/SOURCES/004-test.patch b/SOURCES/004-test.patch deleted file mode 100644 index e17850b..0000000 --- a/SOURCES/004-test.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 7ed7675615ada7d0be5654e0dcb26de60cf5b5e9 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 22 Jun 2020 20:03:56 -0500 -Subject: [PATCH] Test: scheduler: explicitly disable concurrent-fencing in - on_fail_demote4 - -... so the expected output is the same regardless of what default the build was -compiled with ---- - cts/scheduler/on_fail_demote4.xml | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/cts/scheduler/on_fail_demote4.xml b/cts/scheduler/on_fail_demote4.xml -index eb4c4cc..1082266 100644 ---- a/cts/scheduler/on_fail_demote4.xml -+++ b/cts/scheduler/on_fail_demote4.xml -@@ -8,6 +8,7 @@ - - - -+ - - - --- -1.8.3.1 - diff --git a/SOURCES/005-sysconfig.patch b/SOURCES/005-sysconfig.patch deleted file mode 100644 index 4e49cab..0000000 --- a/SOURCES/005-sysconfig.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 85040eb19b9405464b01a7e67eb6769d2a03c611 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 19 Jun 2020 17:49:22 -0500 -Subject: [PATCH] Doc: sysconfig: remove outdated reference to wildcards in - PCMK_trace_files - -Wildcards stopped working when the log filtering implementation changed in -1.1.8 to support PCMK_trace_tags. It's not worth the effort to fix at this -point, so just update the comment in the sysconfig file. ---- - daemons/pacemakerd/pacemaker.sysconfig | 5 ++--- - 1 file changed, 2 insertions(+), 3 deletions(-) - -diff --git a/daemons/pacemakerd/pacemaker.sysconfig b/daemons/pacemakerd/pacemaker.sysconfig -index c7745d8..e4a5c4d 100644 ---- a/daemons/pacemakerd/pacemaker.sysconfig -+++ b/daemons/pacemakerd/pacemaker.sysconfig -@@ -34,9 +34,8 @@ - # Log all messages from a comma-separated list of functions. - # PCMK_trace_functions=function1,function2,function3 - --# Log all messages from a comma-separated list of files (no path). --# Wildcards are supported, e.g. PCMK_trace_files=prefix*.c --# PCMK_trace_files=file.c,other.h -+# Log all messages from a comma-separated list of file names (without path). -+# PCMK_trace_files=file1.c,file2.c - - # Log all messages matching comma-separated list of formats. - # PCMK_trace_formats="Sent delete %d" --- -1.8.3.1 - diff --git a/SOURCES/006-ipc_refactor.patch b/SOURCES/006-ipc_refactor.patch deleted file mode 100644 index 55e7b95..0000000 --- a/SOURCES/006-ipc_refactor.patch +++ /dev/null @@ -1,5198 +0,0 @@ -From f0cfb7b02202b832f5655ee80580d053638e0be1 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 3 Apr 2020 10:27:29 -0500 -Subject: [PATCH 1/8] Refactor: libcrmcommon: combine IPC internal headers - -Combine ipcs_internal.h into ipc_internal.h. ---- - daemons/attrd/attrd_utils.c | 2 +- - daemons/attrd/pacemaker-attrd.c | 2 +- - daemons/based/based_messages.c | 2 +- - daemons/based/based_remote.c | 2 +- - daemons/based/pacemaker-based.h | 2 +- - daemons/controld/controld_control.c | 2 +- - daemons/controld/controld_fsa.h | 2 +- - daemons/controld/controld_messages.c | 2 +- - daemons/controld/controld_messages.h | 2 +- - daemons/execd/execd_alerts.c | 2 +- - daemons/execd/execd_commands.c | 2 +- - daemons/execd/pacemaker-execd.c | 2 +- - daemons/execd/pacemaker-execd.h | 2 +- - daemons/execd/remoted_proxy.c | 2 +- - daemons/fenced/fenced_commands.c | 2 +- - daemons/fenced/fenced_history.c | 2 +- - daemons/fenced/fenced_remote.c | 2 +- - daemons/fenced/pacemaker-fenced.c | 2 +- - daemons/pacemakerd/pacemakerd.c | 4 +- - daemons/schedulerd/pacemaker-schedulerd.c | 2 +- - include/crm/cib/internal.h | 2 +- - include/crm/common/Makefile.am | 2 +- - include/crm/common/ipc_internal.h | 144 ++++++++++++++++++++++++++++- - include/crm/common/ipcs_internal.h | 149 ------------------------------ - include/crm_internal.h | 2 +- - lib/cib/cib_remote.c | 2 +- - lib/common/ipc.c | 4 +- - lib/common/mainloop.c | 2 +- - lib/common/remote.c | 2 +- - lib/lrmd/lrmd_client.c | 2 +- - lib/pacemaker/pcmk_sched_messages.c | 2 +- - maint/mocked/based.c | 2 +- - maint/mocked/based.h | 2 +- - 33 files changed, 172 insertions(+), 187 deletions(-) - delete mode 100644 include/crm/common/ipcs_internal.h - -diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c -index b60b452..c311ddc 100644 ---- a/daemons/attrd/attrd_utils.c -+++ b/daemons/attrd/attrd_utils.c -@@ -17,7 +17,7 @@ - #include - - #include --#include -+#include - #include - - #include "pacemaker-attrd.h" -diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c -index 61e5493..0e944ed 100644 ---- a/daemons/attrd/pacemaker-attrd.c -+++ b/daemons/attrd/pacemaker-attrd.c -@@ -25,7 +25,7 @@ - #include - #include - #include --#include -+#include - #include - #include - -diff --git a/daemons/based/based_messages.c b/daemons/based/based_messages.c -index 4438e28..af0a3a2 100644 ---- a/daemons/based/based_messages.c -+++ b/daemons/based/based_messages.c -@@ -24,7 +24,7 @@ - #include - - #include --#include -+#include - #include - - #include -diff --git a/daemons/based/based_remote.c b/daemons/based/based_remote.c -index ca75b73..70261c3 100644 ---- a/daemons/based/based_remote.c -+++ b/daemons/based/based_remote.c -@@ -26,7 +26,7 @@ - - #include - #include --#include -+#include - #include - #include - #include -diff --git a/daemons/based/pacemaker-based.h b/daemons/based/pacemaker-based.h -index 0d7a5b9..c88ce7c 100644 ---- a/daemons/based/pacemaker-based.h -+++ b/daemons/based/pacemaker-based.h -@@ -22,7 +22,7 @@ - #include - #include - #include --#include -+#include - #include - #include - -diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c -index 1ddcada..7d29205 100644 ---- a/daemons/controld/controld_control.c -+++ b/daemons/controld/controld_control.c -@@ -18,7 +18,7 @@ - #include - #include - #include --#include -+#include - - #include - -diff --git a/daemons/controld/controld_fsa.h b/daemons/controld/controld_fsa.h -index b76a7d2..28eea56 100644 ---- a/daemons/controld/controld_fsa.h -+++ b/daemons/controld/controld_fsa.h -@@ -16,7 +16,7 @@ - # include - # include - # include --# include -+# include - - /*! States the controller can be in */ - enum crmd_fsa_state { -diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c -index 62719ad..59b2069 100644 ---- a/daemons/controld/controld_messages.c -+++ b/daemons/controld/controld_messages.c -@@ -18,7 +18,7 @@ - #include - #include - #include --#include -+#include - - #include - -diff --git a/daemons/controld/controld_messages.h b/daemons/controld/controld_messages.h -index db3ade3..4018deb 100644 ---- a/daemons/controld/controld_messages.h -+++ b/daemons/controld/controld_messages.h -@@ -11,7 +11,7 @@ - # define XML_CRM_MESSAGES__H - - # include --# include -+# include - # include - # include - # include -diff --git a/daemons/execd/execd_alerts.c b/daemons/execd/execd_alerts.c -index 10eca36..ffe60b5 100644 ---- a/daemons/execd/execd_alerts.c -+++ b/daemons/execd/execd_alerts.c -@@ -14,7 +14,7 @@ - #include - #include - #include --#include -+#include - #include - #include - -diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c -index 4d0e457..8da63b4 100644 ---- a/daemons/execd/execd_commands.c -+++ b/daemons/execd/execd_commands.c -@@ -25,7 +25,7 @@ - #include - #include - #include --#include -+#include - #include - - #include "pacemaker-execd.h" -diff --git a/daemons/execd/pacemaker-execd.c b/daemons/execd/pacemaker-execd.c -index df27d1a..c06da7a 100644 ---- a/daemons/execd/pacemaker-execd.c -+++ b/daemons/execd/pacemaker-execd.c -@@ -18,7 +18,7 @@ - #include - #include - #include --#include -+#include - #include - #include - -diff --git a/daemons/execd/pacemaker-execd.h b/daemons/execd/pacemaker-execd.h -index 7ba3e78..d86894b 100644 ---- a/daemons/execd/pacemaker-execd.h -+++ b/daemons/execd/pacemaker-execd.h -@@ -11,7 +11,7 @@ - # define PACEMAKER_EXECD__H - - # include --# include -+# include - # include - # include - -diff --git a/daemons/execd/remoted_proxy.c b/daemons/execd/remoted_proxy.c -index ef0d108..dda7eed 100644 ---- a/daemons/execd/remoted_proxy.c -+++ b/daemons/execd/remoted_proxy.c -@@ -18,7 +18,7 @@ - #include - #include - #include --#include -+#include - #include - #include - -diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c -index d13d1a1..5df472f 100644 ---- a/daemons/fenced/fenced_commands.c -+++ b/daemons/fenced/fenced_commands.c -@@ -25,7 +25,7 @@ - #include - #include - #include --#include -+#include - #include - #include - -diff --git a/daemons/fenced/fenced_history.c b/daemons/fenced/fenced_history.c -index 710d6fe..b48662c 100644 ---- a/daemons/fenced/fenced_history.c -+++ b/daemons/fenced/fenced_history.c -@@ -16,7 +16,7 @@ - #include - #include - #include --#include -+#include - #include - - #include -diff --git a/daemons/fenced/fenced_remote.c b/daemons/fenced/fenced_remote.c -index b89c40a..bf24acb 100644 ---- a/daemons/fenced/fenced_remote.c -+++ b/daemons/fenced/fenced_remote.c -@@ -26,7 +26,7 @@ - #include - #include - #include --#include -+#include - #include - - #include -diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c -index 450814c..6a2935a 100644 ---- a/daemons/fenced/pacemaker-fenced.c -+++ b/daemons/fenced/pacemaker-fenced.c -@@ -24,7 +24,7 @@ - #include - #include - #include --#include -+#include - #include - - #include -diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c -index 5ed4626..64c30e2 100644 ---- a/daemons/pacemakerd/pacemakerd.c -+++ b/daemons/pacemakerd/pacemakerd.c -@@ -24,13 +24,11 @@ - #include /* indirectly: CRM_EX_* */ - #include /* cib_channel_ro */ - #include --#include -+#include - #include - #include - #include - --#include /* PCMK__SPECIAL_PID*, ... */ -- - #ifdef SUPPORT_COROSYNC - #include - #endif -diff --git a/daemons/schedulerd/pacemaker-schedulerd.c b/daemons/schedulerd/pacemaker-schedulerd.c -index 0146ca2..885386d 100644 ---- a/daemons/schedulerd/pacemaker-schedulerd.c -+++ b/daemons/schedulerd/pacemaker-schedulerd.c -@@ -21,7 +21,7 @@ - - #include - --#include -+#include - #include - #include - #include -diff --git a/include/crm/cib/internal.h b/include/crm/cib/internal.h -index df16280..b43cf08 100644 ---- a/include/crm/cib/internal.h -+++ b/include/crm/cib/internal.h -@@ -10,7 +10,7 @@ - #ifndef CIB_INTERNAL__H - # define CIB_INTERNAL__H - # include --# include -+# include - - # define CIB_OP_SLAVE "cib_slave" - # define CIB_OP_SLAVEALL "cib_slave_all" -diff --git a/include/crm/common/Makefile.am b/include/crm/common/Makefile.am -index b38a5c5..776e4a7 100644 ---- a/include/crm/common/Makefile.am -+++ b/include/crm/common/Makefile.am -@@ -13,7 +13,7 @@ headerdir=$(pkgincludedir)/crm/common - - header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h results.h \ - nvpair.h acl.h --noinst_HEADERS = ipcs_internal.h internal.h alerts_internal.h \ -+noinst_HEADERS = internal.h alerts_internal.h \ - iso8601_internal.h remote_internal.h xml_internal.h \ - ipc_internal.h output.h cmdline_internal.h curses_internal.h \ - attrd_internal.h options_internal.h -diff --git a/include/crm/common/ipc_internal.h b/include/crm/common/ipc_internal.h -index 7113d78..a85210d 100644 ---- a/include/crm/common/ipc_internal.h -+++ b/include/crm/common/ipc_internal.h -@@ -1,5 +1,5 @@ - /* -- * Copyright 2019 the Pacemaker project contributors -+ * Copyright 2013-2020 the Pacemaker project contributors - * - * The version control history for this file may have further details. - * -@@ -10,10 +10,26 @@ - #ifndef PCMK__IPC_INTERNAL_H - #define PCMK__IPC_INTERNAL_H - --#include -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#include // bool -+#include // uint32_t -+#include // struct iovec -+#include // uid_t, gid_t, pid_t, size_t -+ -+#ifdef HAVE_GNUTLS_GNUTLS_H -+# include // gnutls_session_t -+#endif - --#include /* US_AUTH_GETPEEREID */ -+#include // guint, gpointer, GQueue, ... -+#include // xmlNode -+#include // qb_ipcs_connection_t, ... - -+#include // US_AUTH_GETPEEREID -+#include -+#include // mainloop_io_t - - /* denotes "non yieldable PID" on FreeBSD, or actual PID1 in scenarios that - require a delicate handling anyway (socket-based activation with systemd); -@@ -69,4 +85,126 @@ - int pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid, - gid_t refgid, pid_t *gotpid); - -+typedef struct pcmk__client_s pcmk__client_t; -+ -+enum pcmk__client_type { -+ PCMK__CLIENT_IPC = 1, -+ PCMK__CLIENT_TCP = 2, -+# ifdef HAVE_GNUTLS_GNUTLS_H -+ PCMK__CLIENT_TLS = 3, -+# endif -+}; -+ -+struct pcmk__remote_s { -+ /* Shared */ -+ char *buffer; -+ size_t buffer_size; -+ size_t buffer_offset; -+ int auth_timeout; -+ int tcp_socket; -+ mainloop_io_t *source; -+ -+ /* CIB-only */ -+ bool authenticated; -+ char *token; -+ -+ /* TLS only */ -+# ifdef HAVE_GNUTLS_GNUTLS_H -+ gnutls_session_t *tls_session; -+ bool tls_handshake_complete; -+# endif -+}; -+ -+enum pcmk__client_flags { -+ pcmk__client_proxied = 0x00001, /* ipc_proxy code only */ -+ pcmk__client_privileged = 0x00002, /* root or cluster user */ -+}; -+ -+struct pcmk__client_s { -+ unsigned int pid; -+ -+ uid_t uid; -+ gid_t gid; -+ -+ char *id; -+ char *name; -+ char *user; -+ -+ /* Provided for server use (not used by library) */ -+ /* @TODO merge options, flags, and kind (reserving lower bits for server) */ -+ long long options; -+ -+ int request_id; -+ uint32_t flags; -+ void *userdata; -+ -+ int event_timer; -+ GQueue *event_queue; -+ -+ /* Depending on the value of kind, only some of the following -+ * will be populated/valid -+ */ -+ enum pcmk__client_type kind; -+ -+ qb_ipcs_connection_t *ipcs; /* IPC */ -+ -+ struct pcmk__remote_s *remote; /* TCP/TLS */ -+ -+ unsigned int queue_backlog; /* IPC queue length after last flush */ -+ unsigned int queue_max; /* Evict client whose queue grows this big */ -+}; -+ -+guint pcmk__ipc_client_count(void); -+void pcmk__foreach_ipc_client(GHFunc func, gpointer user_data); -+void pcmk__foreach_ipc_client_remove(GHRFunc func, gpointer user_data); -+ -+void pcmk__client_cleanup(void); -+ -+pcmk__client_t *pcmk__find_client(qb_ipcs_connection_t *c); -+pcmk__client_t *pcmk__find_client_by_id(const char *id); -+const char *pcmk__client_name(pcmk__client_t *c); -+const char *pcmk__client_type_str(enum pcmk__client_type client_type); -+ -+pcmk__client_t *pcmk__new_unauth_client(void *key); -+pcmk__client_t *pcmk__new_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid); -+void pcmk__free_client(pcmk__client_t *c); -+void pcmk__drop_all_clients(qb_ipcs_service_t *s); -+bool pcmk__set_client_queue_max(pcmk__client_t *client, const char *qmax); -+ -+void pcmk__ipc_send_ack_as(const char *function, int line, pcmk__client_t *c, -+ uint32_t request, uint32_t flags, const char *tag); -+#define pcmk__ipc_send_ack(c, req, flags, tag) \ -+ pcmk__ipc_send_ack_as(__FUNCTION__, __LINE__, (c), (req), (flags), (tag)) -+ -+int pcmk__ipc_prepare_iov(uint32_t request, xmlNode *message, -+ uint32_t max_send_size, -+ struct iovec **result, ssize_t *bytes); -+int pcmk__ipc_send_xml(pcmk__client_t *c, uint32_t request, xmlNode *message, -+ uint32_t flags); -+int pcmk__ipc_send_iov(pcmk__client_t *c, struct iovec *iov, uint32_t flags); -+xmlNode *pcmk__client_data2xml(pcmk__client_t *c, void *data, -+ uint32_t *id, uint32_t *flags); -+ -+int pcmk__client_pid(qb_ipcs_connection_t *c); -+ -+void pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs, -+ struct qb_ipcs_service_handlers *cb); -+void pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs, -+ struct qb_ipcs_service_handlers *cb); -+qb_ipcs_service_t *pcmk__serve_controld_ipc(struct qb_ipcs_service_handlers *cb); -+ -+void pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro, -+ qb_ipcs_service_t **ipcs_rw, -+ qb_ipcs_service_t **ipcs_shm, -+ struct qb_ipcs_service_handlers *ro_cb, -+ struct qb_ipcs_service_handlers *rw_cb); -+ -+void pcmk__stop_based_ipc(qb_ipcs_service_t *ipcs_ro, -+ qb_ipcs_service_t *ipcs_rw, -+ qb_ipcs_service_t *ipcs_shm); -+ -+#ifdef __cplusplus -+} -+#endif -+ - #endif -diff --git a/include/crm/common/ipcs_internal.h b/include/crm/common/ipcs_internal.h -deleted file mode 100644 -index c631dfc..0000000 ---- a/include/crm/common/ipcs_internal.h -+++ /dev/null -@@ -1,149 +0,0 @@ --/* -- * Copyright 2013-2020 the Pacemaker project contributors -- * -- * The version control history for this file may have further details. -- * -- * This source code is licensed under the GNU Lesser General Public License -- * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -- */ -- --#ifndef CRM_COMMON_IPCS__H --# define CRM_COMMON_IPCS__H -- --#ifdef __cplusplus --extern "C" { --#endif -- --# include --# include --# ifdef HAVE_GNUTLS_GNUTLS_H --# undef KEYFILE --# include --# endif -- --# include --# include -- --typedef struct pcmk__client_s pcmk__client_t; -- --enum pcmk__client_type { -- PCMK__CLIENT_IPC = 1, -- PCMK__CLIENT_TCP = 2, --# ifdef HAVE_GNUTLS_GNUTLS_H -- PCMK__CLIENT_TLS = 3, --# endif --}; -- --struct pcmk__remote_s { -- /* Shared */ -- char *buffer; -- size_t buffer_size; -- size_t buffer_offset; -- int auth_timeout; -- int tcp_socket; -- mainloop_io_t *source; -- -- /* CIB-only */ -- bool authenticated; -- char *token; -- -- /* TLS only */ --# ifdef HAVE_GNUTLS_GNUTLS_H -- gnutls_session_t *tls_session; -- bool tls_handshake_complete; --# endif --}; -- --enum pcmk__client_flags { -- pcmk__client_proxied = 0x00001, /* ipc_proxy code only */ -- pcmk__client_privileged = 0x00002, /* root or cluster user */ --}; -- --struct pcmk__client_s { -- uint pid; -- -- uid_t uid; -- gid_t gid; -- -- char *id; -- char *name; -- char *user; -- -- /* Provided for server use (not used by library) */ -- /* @TODO merge options, flags, and kind (reserving lower bits for server) */ -- long long options; -- -- int request_id; -- uint32_t flags; -- void *userdata; -- -- int event_timer; -- GQueue *event_queue; -- -- /* Depending on the value of kind, only some of the following -- * will be populated/valid -- */ -- enum pcmk__client_type kind; -- -- qb_ipcs_connection_t *ipcs; /* IPC */ -- -- struct pcmk__remote_s *remote; /* TCP/TLS */ -- -- unsigned int queue_backlog; /* IPC queue length after last flush */ -- unsigned int queue_max; /* Evict client whose queue grows this big */ --}; -- --guint pcmk__ipc_client_count(void); --void pcmk__foreach_ipc_client(GHFunc func, gpointer user_data); --void pcmk__foreach_ipc_client_remove(GHRFunc func, gpointer user_data); -- --void pcmk__client_cleanup(void); -- --pcmk__client_t *pcmk__find_client(qb_ipcs_connection_t *c); --pcmk__client_t *pcmk__find_client_by_id(const char *id); --const char *pcmk__client_name(pcmk__client_t *c); --const char *pcmk__client_type_str(enum pcmk__client_type client_type); -- --pcmk__client_t *pcmk__new_unauth_client(void *key); --pcmk__client_t *pcmk__new_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid); --void pcmk__free_client(pcmk__client_t *c); --void pcmk__drop_all_clients(qb_ipcs_service_t *s); --bool pcmk__set_client_queue_max(pcmk__client_t *client, const char *qmax); -- --void pcmk__ipc_send_ack_as(const char *function, int line, pcmk__client_t *c, -- uint32_t request, uint32_t flags, const char *tag); --#define pcmk__ipc_send_ack(c, req, flags, tag) \ -- pcmk__ipc_send_ack_as(__FUNCTION__, __LINE__, (c), (req), (flags), (tag)) -- --int pcmk__ipc_prepare_iov(uint32_t request, xmlNode *message, -- uint32_t max_send_size, -- struct iovec **result, ssize_t *bytes); --int pcmk__ipc_send_xml(pcmk__client_t *c, uint32_t request, xmlNode *message, -- uint32_t flags); --int pcmk__ipc_send_iov(pcmk__client_t *c, struct iovec *iov, uint32_t flags); --xmlNode *pcmk__client_data2xml(pcmk__client_t *c, void *data, -- uint32_t *id, uint32_t *flags); -- --int pcmk__client_pid(qb_ipcs_connection_t *c); -- --void pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs, -- struct qb_ipcs_service_handlers *cb); --void pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs, -- struct qb_ipcs_service_handlers *cb); --qb_ipcs_service_t *pcmk__serve_controld_ipc(struct qb_ipcs_service_handlers *cb); -- --void pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro, -- qb_ipcs_service_t **ipcs_rw, -- qb_ipcs_service_t **ipcs_shm, -- struct qb_ipcs_service_handlers *ro_cb, -- struct qb_ipcs_service_handlers *rw_cb); -- --void pcmk__stop_based_ipc(qb_ipcs_service_t *ipcs_ro, -- qb_ipcs_service_t *ipcs_rw, -- qb_ipcs_service_t *ipcs_shm); -- --#ifdef __cplusplus --} --#endif -- --#endif -diff --git a/include/crm_internal.h b/include/crm_internal.h -index 882cad8..15f9d2b 100644 ---- a/include/crm_internal.h -+++ b/include/crm_internal.h -@@ -26,7 +26,7 @@ - - # include - # include --# include -+# include - # include - # include - -diff --git a/lib/cib/cib_remote.c b/lib/cib/cib_remote.c -index ed93700..a011810 100644 ---- a/lib/cib/cib_remote.c -+++ b/lib/cib/cib_remote.c -@@ -23,7 +23,7 @@ - #include - #include - #include --#include -+#include - #include - #include - -diff --git a/lib/common/ipc.c b/lib/common/ipc.c -index 34bd594..defaa7e 100644 ---- a/lib/common/ipc.c -+++ b/lib/common/ipc.c -@@ -35,9 +35,7 @@ - #include /* indirectly: pcmk_err_generic */ - #include - #include --#include -- --#include /* PCMK__SPECIAL_PID* */ -+#include - - #define PCMK_IPC_VERSION 1 - -diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c -index e3640f5..10450e4 100644 ---- a/lib/common/mainloop.c -+++ b/lib/common/mainloop.c -@@ -23,7 +23,7 @@ - #include - #include - #include --#include -+#include - - #include - -diff --git a/lib/common/remote.c b/lib/common/remote.c -index 94e06dd..3a97e54 100644 ---- a/lib/common/remote.c -+++ b/lib/common/remote.c -@@ -28,7 +28,7 @@ - #include - #include - --#include -+#include - #include - #include - #include -diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c -index a2c7200..a6c023b 100644 ---- a/lib/lrmd/lrmd_client.c -+++ b/lib/lrmd/lrmd_client.c -@@ -27,7 +27,7 @@ - #include - #include - #include --#include -+#include - #include - #include - -diff --git a/lib/pacemaker/pcmk_sched_messages.c b/lib/pacemaker/pcmk_sched_messages.c -index 9d013be..3d09a5e 100644 ---- a/lib/pacemaker/pcmk_sched_messages.c -+++ b/lib/pacemaker/pcmk_sched_messages.c -@@ -20,7 +20,7 @@ - - #include - #include --#include -+#include - - gboolean show_scores = FALSE; - gboolean show_utilization = FALSE; -diff --git a/maint/mocked/based.c b/maint/mocked/based.c -index 0d5fd2d..2cfad9f 100644 ---- a/maint/mocked/based.c -+++ b/maint/mocked/based.c -@@ -23,7 +23,7 @@ - - #include - #if 0 --#include "crm/common/ipcs_internal.h" /* pcmk__client_t */ -+#include "crm/common/ipc_internal.h" /* pcmk__client_t */ - #include "crm/common/xml.h" /* crm_xml_add */ - #endif - #include "crm/msg_xml.h" /* F_SUBTYPE */ -diff --git a/maint/mocked/based.h b/maint/mocked/based.h -index ef1dc95..c214c08 100644 ---- a/maint/mocked/based.h -+++ b/maint/mocked/based.h -@@ -11,7 +11,7 @@ - #include /* size_t */ - #include /* bool */ - --#include /* pcmk__client_t */ -+#include /* pcmk__client_t */ - - - struct module_s; --- -1.8.3.1 - - -From 6743aee47c7b4cfa107ef02512fda7bebdd29efc Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 2 Apr 2020 17:01:50 -0500 -Subject: [PATCH 2/8] Refactor: libcrmcommon: separate IPC code into multiple - files - -A significant amount of new IPC code will be added soon, so avoid the file -sizes becoming ridiculous. - -Before: - 210 include/crm/common/ipc_internal.h - 1846 lib/common/ipc.c - 2056 total - -After: - 215 include/crm/common/ipc_internal.h - 755 lib/common/ipc_client.c - 103 lib/common/ipc_common.c - 903 lib/common/ipc_server.c - 146 lib/common/messages.c - 2122 total - -Rename newly exposed symbols per current style guidelines. ---- - include/crm/common/ipc.h | 25 +- - include/crm/common/ipc_internal.h | 5 + - lib/common/Makefile.am | 5 +- - lib/common/crmcommon_private.h | 37 +- - lib/common/ipc.c | 1846 ------------------------------------- - lib/common/ipc_client.c | 755 +++++++++++++++ - lib/common/ipc_common.c | 103 +++ - lib/common/ipc_server.c | 903 ++++++++++++++++++ - lib/common/messages.c | 146 +++ - 9 files changed, 1969 insertions(+), 1856 deletions(-) - delete mode 100644 lib/common/ipc.c - create mode 100644 lib/common/ipc_client.c - create mode 100644 lib/common/ipc_common.c - create mode 100644 lib/common/ipc_server.c - create mode 100644 lib/common/messages.c - -diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h -index 79763f6..a0df956 100644 ---- a/include/crm/common/ipc.h -+++ b/include/crm/common/ipc.h -@@ -24,17 +24,30 @@ extern "C" { - #include - #include - --/* clplumbing based IPC */ -+/* -+ * Message creation utilities -+ * -+ * These are used for both IPC messages and cluster layer messages. However, -+ * since this is public API, they stay in this header for backward -+ * compatibility. -+ */ - --# define create_reply(request, xml_response_data) create_reply_adv(request, xml_response_data, __FUNCTION__); --xmlNode *create_reply_adv(xmlNode * request, xmlNode * xml_response_data, const char *origin); -+#define create_reply(request, xml_response_data) \ -+ create_reply_adv(request, xml_response_data, __FUNCTION__) - --# define create_request(task, xml_data, host_to, sys_to, sys_from, uuid_from) create_request_adv(task, xml_data, host_to, sys_to, sys_from, uuid_from, __FUNCTION__) -+xmlNode *create_reply_adv(xmlNode *request, xmlNode *xml_response_data, -+ const char *origin); - --xmlNode *create_request_adv(const char *task, xmlNode * xml_data, const char *host_to, -- const char *sys_to, const char *sys_from, const char *uuid_from, -+#define create_request(task, xml_data, host_to, sys_to, sys_from, uuid_from) \ -+ create_request_adv(task, xml_data, host_to, sys_to, sys_from, uuid_from, \ -+ __FUNCTION__) -+ -+xmlNode *create_request_adv(const char *task, xmlNode *xml_data, -+ const char *host_to, const char *sys_to, -+ const char *sys_from, const char *uuid_from, - const char *origin); - -+ - /* *INDENT-OFF* */ - enum crm_ipc_flags - { -diff --git a/include/crm/common/ipc_internal.h b/include/crm/common/ipc_internal.h -index a85210d..6a1fcf3 100644 ---- a/include/crm/common/ipc_internal.h -+++ b/include/crm/common/ipc_internal.h -@@ -85,6 +85,11 @@ extern "C" { - int pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid, - gid_t refgid, pid_t *gotpid); - -+ -+/* -+ * Server-related -+ */ -+ - typedef struct pcmk__client_s pcmk__client_t; - - enum pcmk__client_type { -diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am -index fae17f5..29404a6 100644 ---- a/lib/common/Makefile.am -+++ b/lib/common/Makefile.am -@@ -47,10 +47,13 @@ endif - libcrmcommon_la_SOURCES += cmdline.c - libcrmcommon_la_SOURCES += digest.c - libcrmcommon_la_SOURCES += io.c --libcrmcommon_la_SOURCES += ipc.c -+libcrmcommon_la_SOURCES += ipc_client.c -+libcrmcommon_la_SOURCES += ipc_common.c -+libcrmcommon_la_SOURCES += ipc_server.c - libcrmcommon_la_SOURCES += iso8601.c - libcrmcommon_la_SOURCES += logging.c - libcrmcommon_la_SOURCES += mainloop.c -+libcrmcommon_la_SOURCES += messages.c - libcrmcommon_la_SOURCES += nvpair.c - libcrmcommon_la_SOURCES += operations.c - libcrmcommon_la_SOURCES += options.c -diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h -index dfb1e54..d06fa20 100644 ---- a/lib/common/crmcommon_private.h -+++ b/lib/common/crmcommon_private.h -@@ -1,5 +1,5 @@ - /* -- * Copyright 2018-2019 the Pacemaker project contributors -+ * Copyright 2018-2020 the Pacemaker project contributors - * - * The version control history for this file may have further details. - * -@@ -14,6 +14,17 @@ - * declared with G_GNUC_INTERNAL for efficiency. - */ - -+#include // uint8_t, uint32_t -+#include // bool -+#include // size_t -+#include // GList -+#include // xmlNode, xmlAttr -+#include // struct qb_ipc_response_header -+ -+/* -+ * XML and ACLs -+ */ -+ - enum xml_private_flags { - xpf_none = 0x0000, - xpf_dirty = 0x0001, -@@ -40,8 +51,8 @@ typedef struct xml_private_s { - long check; - uint32_t flags; - char *user; -- GListPtr acls; -- GListPtr deleted_objs; -+ GList *acls; -+ GList *deleted_objs; - } xml_private_t; - - G_GNUC_INTERNAL -@@ -86,4 +97,24 @@ pcmk__xml_attr_value(const xmlAttr *attr) - : (const char *) attr->children->content; - } - -+/* -+ * IPC -+ */ -+ -+#define PCMK__IPC_VERSION 1 -+ -+typedef struct pcmk__ipc_header_s { -+ struct qb_ipc_response_header qb; -+ uint32_t size_uncompressed; -+ uint32_t size_compressed; -+ uint32_t flags; -+ uint8_t version; -+} pcmk__ipc_header_t; -+ -+G_GNUC_INTERNAL -+unsigned int pcmk__ipc_buffer_size(unsigned int max); -+ -+G_GNUC_INTERNAL -+bool pcmk__valid_ipc_header(const pcmk__ipc_header_t *header); -+ - #endif // CRMCOMMON_PRIVATE__H -diff --git a/lib/common/ipc.c b/lib/common/ipc.c -deleted file mode 100644 -index defaa7e..0000000 ---- a/lib/common/ipc.c -+++ /dev/null -@@ -1,1846 +0,0 @@ --/* -- * Copyright 2004-2020 the Pacemaker project contributors -- * -- * The version control history for this file may have further details. -- * -- * This source code is licensed under the GNU Lesser General Public License -- * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -- */ -- --#include -- --#if defined(US_AUTH_PEERCRED_UCRED) || defined(US_AUTH_PEERCRED_SOCKPEERCRED) --# ifdef US_AUTH_PEERCRED_UCRED --# ifndef _GNU_SOURCE --# define _GNU_SOURCE --# endif --# endif --# include --#elif defined(US_AUTH_GETPEERUCRED) --# include --#endif -- --#include -- --#include --#include --#include --#include --#include -- --#include --#include --#include -- --#include /* indirectly: pcmk_err_generic */ --#include --#include --#include -- --#define PCMK_IPC_VERSION 1 -- --/* Evict clients whose event queue grows this large (by default) */ --#define PCMK_IPC_DEFAULT_QUEUE_MAX 500 -- --struct crm_ipc_response_header { -- struct qb_ipc_response_header qb; -- uint32_t size_uncompressed; -- uint32_t size_compressed; -- uint32_t flags; -- uint8_t version; /* Protect against version changes for anyone that might bother to statically link us */ --}; -- --static int hdr_offset = 0; --static unsigned int ipc_buffer_max = 0; --static unsigned int pick_ipc_buffer(unsigned int max); -- --static inline void --crm_ipc_init(void) --{ -- if (hdr_offset == 0) { -- hdr_offset = sizeof(struct crm_ipc_response_header); -- } -- if (ipc_buffer_max == 0) { -- ipc_buffer_max = pick_ipc_buffer(0); -- } --} -- --unsigned int --crm_ipc_default_buffer_size(void) --{ -- return pick_ipc_buffer(0); --} -- --static char * --generateReference(const char *custom1, const char *custom2) --{ -- static uint ref_counter = 0; -- -- return crm_strdup_printf("%s-%s-%lld-%u", -- (custom1? custom1 : "_empty_"), -- (custom2? custom2 : "_empty_"), -- (long long) time(NULL), ref_counter++); --} -- --xmlNode * --create_request_adv(const char *task, xmlNode * msg_data, -- const char *host_to, const char *sys_to, -- const char *sys_from, const char *uuid_from, const char *origin) --{ -- char *true_from = NULL; -- xmlNode *request = NULL; -- char *reference = generateReference(task, sys_from); -- -- if (uuid_from != NULL) { -- true_from = generate_hash_key(sys_from, uuid_from); -- } else if (sys_from != NULL) { -- true_from = strdup(sys_from); -- } else { -- crm_err("No sys from specified"); -- } -- -- // host_from will get set for us if necessary by the controller when routed -- request = create_xml_node(NULL, __FUNCTION__); -- crm_xml_add(request, F_CRM_ORIGIN, origin); -- crm_xml_add(request, F_TYPE, T_CRM); -- crm_xml_add(request, F_CRM_VERSION, CRM_FEATURE_SET); -- crm_xml_add(request, F_CRM_MSG_TYPE, XML_ATTR_REQUEST); -- crm_xml_add(request, F_CRM_REFERENCE, reference); -- crm_xml_add(request, F_CRM_TASK, task); -- crm_xml_add(request, F_CRM_SYS_TO, sys_to); -- crm_xml_add(request, F_CRM_SYS_FROM, true_from); -- -- /* HOSTTO will be ignored if it is to the DC anyway. */ -- if (host_to != NULL && strlen(host_to) > 0) { -- crm_xml_add(request, F_CRM_HOST_TO, host_to); -- } -- -- if (msg_data != NULL) { -- add_message_xml(request, F_CRM_DATA, msg_data); -- } -- free(reference); -- free(true_from); -- -- return request; --} -- --/* -- * This method adds a copy of xml_response_data -- */ --xmlNode * --create_reply_adv(xmlNode * original_request, xmlNode * xml_response_data, const char *origin) --{ -- xmlNode *reply = NULL; -- -- const char *host_from = crm_element_value(original_request, F_CRM_HOST_FROM); -- const char *sys_from = crm_element_value(original_request, F_CRM_SYS_FROM); -- const char *sys_to = crm_element_value(original_request, F_CRM_SYS_TO); -- const char *type = crm_element_value(original_request, F_CRM_MSG_TYPE); -- const char *operation = crm_element_value(original_request, F_CRM_TASK); -- const char *crm_msg_reference = crm_element_value(original_request, F_CRM_REFERENCE); -- -- if (type == NULL) { -- crm_err("Cannot create new_message, no message type in original message"); -- CRM_ASSERT(type != NULL); -- return NULL; --#if 0 -- } else if (strcasecmp(XML_ATTR_REQUEST, type) != 0) { -- crm_err("Cannot create new_message, original message was not a request"); -- return NULL; --#endif -- } -- reply = create_xml_node(NULL, __FUNCTION__); -- if (reply == NULL) { -- crm_err("Cannot create new_message, malloc failed"); -- return NULL; -- } -- -- crm_xml_add(reply, F_CRM_ORIGIN, origin); -- crm_xml_add(reply, F_TYPE, T_CRM); -- crm_xml_add(reply, F_CRM_VERSION, CRM_FEATURE_SET); -- crm_xml_add(reply, F_CRM_MSG_TYPE, XML_ATTR_RESPONSE); -- crm_xml_add(reply, F_CRM_REFERENCE, crm_msg_reference); -- crm_xml_add(reply, F_CRM_TASK, operation); -- -- /* since this is a reply, we reverse the from and to */ -- crm_xml_add(reply, F_CRM_SYS_TO, sys_from); -- crm_xml_add(reply, F_CRM_SYS_FROM, sys_to); -- -- /* HOSTTO will be ignored if it is to the DC anyway. */ -- if (host_from != NULL && strlen(host_from) > 0) { -- crm_xml_add(reply, F_CRM_HOST_TO, host_from); -- } -- -- if (xml_response_data != NULL) { -- add_message_xml(reply, F_CRM_DATA, xml_response_data); -- } -- -- return reply; --} -- --/* Libqb based IPC */ -- --/* Server... */ -- --static GHashTable *client_connections = NULL; -- --/*! -- * \internal -- * \brief Count IPC clients -- * -- * \return Number of active IPC client connections -- */ --guint --pcmk__ipc_client_count() --{ -- return client_connections? g_hash_table_size(client_connections) : 0; --} -- --/*! -- * \internal -- * \brief Execute a function for each active IPC client connection -- * -- * \param[in] func Function to call -- * \param[in] user_data Pointer to pass to function -- * -- * \note The parameters are the same as for g_hash_table_foreach(). -- */ --void --pcmk__foreach_ipc_client(GHFunc func, gpointer user_data) --{ -- if ((func != NULL) && (client_connections != NULL)) { -- g_hash_table_foreach(client_connections, func, user_data); -- } --} -- --/*! -- * \internal -- * \brief Remote IPC clients based on iterative function result -- * -- * \param[in] func Function to call for each active IPC client -- * \param[in] user_data Pointer to pass to function -- * -- * \note The parameters are the same as for g_hash_table_foreach_remove(). -- */ --void --pcmk__foreach_ipc_client_remove(GHRFunc func, gpointer user_data) --{ -- if ((func != NULL) && (client_connections != NULL)) { -- g_hash_table_foreach_remove(client_connections, func, user_data); -- } --} -- --pcmk__client_t * --pcmk__find_client(qb_ipcs_connection_t *c) --{ -- if (client_connections) { -- return g_hash_table_lookup(client_connections, c); -- } -- -- crm_trace("No client found for %p", c); -- return NULL; --} -- --pcmk__client_t * --pcmk__find_client_by_id(const char *id) --{ -- gpointer key; -- pcmk__client_t *client; -- GHashTableIter iter; -- -- if (client_connections && id) { -- g_hash_table_iter_init(&iter, client_connections); -- while (g_hash_table_iter_next(&iter, &key, (gpointer *) & client)) { -- if (strcmp(client->id, id) == 0) { -- return client; -- } -- } -- } -- -- crm_trace("No client found with id=%s", id); -- return NULL; --} -- --const char * --pcmk__client_name(pcmk__client_t *c) --{ -- if (c == NULL) { -- return "null"; -- } else if (c->name == NULL && c->id == NULL) { -- return "unknown"; -- } else if (c->name == NULL) { -- return c->id; -- } else { -- return c->name; -- } --} -- --const char * --pcmk__client_type_str(enum pcmk__client_type client_type) --{ -- switch (client_type) { -- case PCMK__CLIENT_IPC: -- return "IPC"; -- case PCMK__CLIENT_TCP: -- return "TCP"; --#ifdef HAVE_GNUTLS_GNUTLS_H -- case PCMK__CLIENT_TLS: -- return "TLS"; --#endif -- default: -- return "unknown"; -- } --} -- --void --pcmk__client_cleanup(void) --{ -- if (client_connections != NULL) { -- int active = g_hash_table_size(client_connections); -- -- if (active) { -- crm_err("Exiting with %d active IPC client%s", -- active, pcmk__plural_s(active)); -- } -- g_hash_table_destroy(client_connections); client_connections = NULL; -- } --} -- --void --pcmk__drop_all_clients(qb_ipcs_service_t *service) --{ -- qb_ipcs_connection_t *c = NULL; -- -- if (service == NULL) { -- return; -- } -- -- c = qb_ipcs_connection_first_get(service); -- -- while (c != NULL) { -- qb_ipcs_connection_t *last = c; -- -- c = qb_ipcs_connection_next_get(service, last); -- -- /* There really shouldn't be anyone connected at this point */ -- crm_notice("Disconnecting client %p, pid=%d...", -- last, pcmk__client_pid(last)); -- qb_ipcs_disconnect(last); -- qb_ipcs_connection_unref(last); -- } --} -- --/*! -- * \internal -- * \brief Allocate a new pcmk__client_t object based on an IPC connection -- * -- * \param[in] c IPC connection (or NULL to allocate generic client) -- * \param[in] key Connection table key (or NULL to use sane default) -- * \param[in] uid_client UID corresponding to c (ignored if c is NULL) -- * -- * \return Pointer to new pcmk__client_t (or NULL on error) -- */ --static pcmk__client_t * --client_from_connection(qb_ipcs_connection_t *c, void *key, uid_t uid_client) --{ -- pcmk__client_t *client = calloc(1, sizeof(pcmk__client_t)); -- -- if (client == NULL) { -- crm_perror(LOG_ERR, "Allocating client"); -- return NULL; -- } -- -- if (c) { --#if ENABLE_ACL -- client->user = pcmk__uid2username(uid_client); -- if (client->user == NULL) { -- client->user = strdup("#unprivileged"); -- CRM_CHECK(client->user != NULL, free(client); return NULL); -- crm_err("Unable to enforce ACLs for user ID %d, assuming unprivileged", -- uid_client); -- } --#endif -- client->ipcs = c; -- client->kind = PCMK__CLIENT_IPC; -- client->pid = pcmk__client_pid(c); -- if (key == NULL) { -- key = c; -- } -- } -- -- client->id = crm_generate_uuid(); -- if (client->id == NULL) { -- crm_err("Could not generate UUID for client"); -- free(client->user); -- free(client); -- return NULL; -- } -- if (key == NULL) { -- key = client->id; -- } -- if (client_connections == NULL) { -- crm_trace("Creating IPC client table"); -- client_connections = g_hash_table_new(g_direct_hash, g_direct_equal); -- } -- g_hash_table_insert(client_connections, key, client); -- return client; --} -- --/*! -- * \brief Allocate a new pcmk__client_t object and generate its ID -- * -- * \param[in] key What to use as connections hash table key (NULL to use ID) -- * -- * \return Pointer to new pcmk__client_t (asserts on failure) -- */ --pcmk__client_t * --pcmk__new_unauth_client(void *key) --{ -- pcmk__client_t *client = client_from_connection(NULL, key, 0); -- -- CRM_ASSERT(client != NULL); -- return client; --} -- --pcmk__client_t * --pcmk__new_client(qb_ipcs_connection_t *c, uid_t uid_client, gid_t gid_client) --{ -- gid_t uid_cluster = 0; -- gid_t gid_cluster = 0; -- -- pcmk__client_t *client = NULL; -- -- CRM_CHECK(c != NULL, return NULL); -- -- if (pcmk_daemon_user(&uid_cluster, &gid_cluster) < 0) { -- static bool need_log = TRUE; -- -- if (need_log) { -- crm_warn("Could not find user and group IDs for user %s", -- CRM_DAEMON_USER); -- need_log = FALSE; -- } -- } -- -- if (uid_client != 0) { -- crm_trace("Giving group %u access to new IPC connection", gid_cluster); -- /* Passing -1 to chown(2) means don't change */ -- qb_ipcs_connection_auth_set(c, -1, gid_cluster, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); -- } -- -- /* TODO: Do our own auth checking, return NULL if unauthorized */ -- client = client_from_connection(c, NULL, uid_client); -- if (client == NULL) { -- return NULL; -- } -- -- if ((uid_client == 0) || (uid_client == uid_cluster)) { -- /* Remember when a connection came from root or hacluster */ -- set_bit(client->flags, pcmk__client_privileged); -- } -- -- crm_debug("New IPC client %s for PID %u with uid %d and gid %d", -- client->id, client->pid, uid_client, gid_client); -- return client; --} -- --static struct iovec * --pcmk__new_ipc_event(void) --{ -- struct iovec *iov = calloc(2, sizeof(struct iovec)); -- -- CRM_ASSERT(iov != NULL); -- return iov; --} -- --/*! -- * \brief Free an I/O vector created by pcmk__ipc_prepare_iov() -- * -- * \param[in] event I/O vector to free -- */ --void --pcmk_free_ipc_event(struct iovec *event) --{ -- if (event != NULL) { -- free(event[0].iov_base); -- free(event[1].iov_base); -- free(event); -- } --} -- --static void --free_event(gpointer data) --{ -- pcmk_free_ipc_event((struct iovec *) data); --} -- --static void --add_event(pcmk__client_t *c, struct iovec *iov) --{ -- if (c->event_queue == NULL) { -- c->event_queue = g_queue_new(); -- } -- g_queue_push_tail(c->event_queue, iov); --} -- --void --pcmk__free_client(pcmk__client_t *c) --{ -- if (c == NULL) { -- return; -- } -- -- if (client_connections) { -- if (c->ipcs) { -- crm_trace("Destroying %p/%p (%d remaining)", -- c, c->ipcs, g_hash_table_size(client_connections) - 1); -- g_hash_table_remove(client_connections, c->ipcs); -- -- } else { -- crm_trace("Destroying remote connection %p (%d remaining)", -- c, g_hash_table_size(client_connections) - 1); -- g_hash_table_remove(client_connections, c->id); -- } -- } -- -- if (c->event_timer) { -- g_source_remove(c->event_timer); -- } -- -- if (c->event_queue) { -- crm_debug("Destroying %d events", g_queue_get_length(c->event_queue)); -- g_queue_free_full(c->event_queue, free_event); -- } -- -- free(c->id); -- free(c->name); -- free(c->user); -- if (c->remote) { -- if (c->remote->auth_timeout) { -- g_source_remove(c->remote->auth_timeout); -- } -- free(c->remote->buffer); -- free(c->remote); -- } -- free(c); --} -- --/*! -- * \internal -- * \brief Raise IPC eviction threshold for a client, if allowed -- * -- * \param[in,out] client Client to modify -- * \param[in] qmax New threshold (as non-NULL string) -- * -- * \return TRUE if change was allowed, FALSE otherwise -- */ --bool --pcmk__set_client_queue_max(pcmk__client_t *client, const char *qmax) --{ -- if (is_set(client->flags, pcmk__client_privileged)) { -- long long qmax_int; -- -- errno = 0; -- qmax_int = crm_parse_ll(qmax, NULL); -- if ((errno == 0) && (qmax_int > 0)) { -- client->queue_max = (unsigned int) qmax_int; -- return TRUE; -- } -- } -- return FALSE; --} -- --int --pcmk__client_pid(qb_ipcs_connection_t *c) --{ -- struct qb_ipcs_connection_stats stats; -- -- stats.client_pid = 0; -- qb_ipcs_connection_stats_get(c, &stats, 0); -- return stats.client_pid; --} -- --/*! -- * \internal -- * \brief Retrieve message XML from data read from client IPC -- * -- * \param[in] c IPC client connection -- * \param[in] data Data read from client connection -- * \param[out] id Where to store message ID from libqb header -- * \param[out] flags Where to store flags from libqb header -- * -- * \return Message XML on success, NULL otherwise -- */ --xmlNode * --pcmk__client_data2xml(pcmk__client_t *c, void *data, uint32_t *id, -- uint32_t *flags) --{ -- xmlNode *xml = NULL; -- char *uncompressed = NULL; -- char *text = ((char *)data) + sizeof(struct crm_ipc_response_header); -- struct crm_ipc_response_header *header = data; -- -- if (id) { -- *id = ((struct qb_ipc_response_header *)data)->id; -- } -- if (flags) { -- *flags = header->flags; -- } -- -- if (is_set(header->flags, crm_ipc_proxied)) { -- /* Mark this client as being the endpoint of a proxy connection. -- * Proxy connections responses are sent on the event channel, to avoid -- * blocking the controller serving as proxy. -- */ -- c->flags |= pcmk__client_proxied; -- } -- -- if(header->version > PCMK_IPC_VERSION) { -- crm_err("Filtering incompatible v%d IPC message, we only support versions <= %d", -- header->version, PCMK_IPC_VERSION); -- return NULL; -- } -- -- if (header->size_compressed) { -- int rc = 0; -- unsigned int size_u = 1 + header->size_uncompressed; -- uncompressed = calloc(1, size_u); -- -- crm_trace("Decompressing message data %u bytes into %u bytes", -- header->size_compressed, size_u); -- -- rc = BZ2_bzBuffToBuffDecompress(uncompressed, &size_u, text, header->size_compressed, 1, 0); -- text = uncompressed; -- -- if (rc != BZ_OK) { -- crm_err("Decompression failed: %s " CRM_XS " bzerror=%d", -- bz2_strerror(rc), rc); -- free(uncompressed); -- return NULL; -- } -- } -- -- CRM_ASSERT(text[header->size_uncompressed - 1] == 0); -- -- xml = string2xml(text); -- crm_log_xml_trace(xml, "[IPC received]"); -- -- free(uncompressed); -- return xml; --} -- --static int crm_ipcs_flush_events(pcmk__client_t *c); -- --static gboolean --crm_ipcs_flush_events_cb(gpointer data) --{ -- pcmk__client_t *c = data; -- -- c->event_timer = 0; -- crm_ipcs_flush_events(c); -- return FALSE; --} -- --/*! -- * \internal -- * \brief Add progressive delay before next event queue flush -- * -- * \param[in,out] c Client connection to add delay to -- * \param[in] queue_len Current event queue length -- */ --static inline void --delay_next_flush(pcmk__client_t *c, unsigned int queue_len) --{ -- /* Delay a maximum of 1.5 seconds */ -- guint delay = (queue_len < 5)? (1000 + 100 * queue_len) : 1500; -- -- c->event_timer = g_timeout_add(delay, crm_ipcs_flush_events_cb, c); --} -- --/*! -- * \internal -- * \brief Send client any messages in its queue -- * -- * \param[in] c Client to flush -- * -- * \return Standard Pacemaker return value -- */ --static int --crm_ipcs_flush_events(pcmk__client_t *c) --{ -- int rc = pcmk_rc_ok; -- ssize_t qb_rc = 0; -- unsigned int sent = 0; -- unsigned int queue_len = 0; -- -- if (c == NULL) { -- return rc; -- -- } else if (c->event_timer) { -- /* There is already a timer, wait until it goes off */ -- crm_trace("Timer active for %p - %d", c->ipcs, c->event_timer); -- return rc; -- } -- -- if (c->event_queue) { -- queue_len = g_queue_get_length(c->event_queue); -- } -- while (sent < 100) { -- struct crm_ipc_response_header *header = NULL; -- struct iovec *event = NULL; -- -- if (c->event_queue) { -- // We don't pop unless send is successful -- event = g_queue_peek_head(c->event_queue); -- } -- if (event == NULL) { // Queue is empty -- break; -- } -- -- qb_rc = qb_ipcs_event_sendv(c->ipcs, event, 2); -- if (qb_rc < 0) { -- rc = (int) -qb_rc; -- break; -- } -- event = g_queue_pop_head(c->event_queue); -- -- sent++; -- header = event[0].iov_base; -- if (header->size_compressed) { -- crm_trace("Event %d to %p[%d] (%lld compressed bytes) sent", -- header->qb.id, c->ipcs, c->pid, (long long) qb_rc); -- } else { -- crm_trace("Event %d to %p[%d] (%lld bytes) sent: %.120s", -- header->qb.id, c->ipcs, c->pid, (long long) qb_rc, -- (char *) (event[1].iov_base)); -- } -- pcmk_free_ipc_event(event); -- } -- -- queue_len -= sent; -- if (sent > 0 || queue_len) { -- crm_trace("Sent %d events (%d remaining) for %p[%d]: %s (%lld)", -- sent, queue_len, c->ipcs, c->pid, -- pcmk_rc_str(rc), (long long) qb_rc); -- } -- -- if (queue_len) { -- -- /* Allow clients to briefly fall behind on processing incoming messages, -- * but drop completely unresponsive clients so the connection doesn't -- * consume resources indefinitely. -- */ -- if (queue_len > QB_MAX(c->queue_max, PCMK_IPC_DEFAULT_QUEUE_MAX)) { -- if ((c->queue_backlog <= 1) || (queue_len < c->queue_backlog)) { -- /* Don't evict for a new or shrinking backlog */ -- crm_warn("Client with process ID %u has a backlog of %u messages " -- CRM_XS " %p", c->pid, queue_len, c->ipcs); -- } else { -- crm_err("Evicting client with process ID %u due to backlog of %u messages " -- CRM_XS " %p", c->pid, queue_len, c->ipcs); -- c->queue_backlog = 0; -- qb_ipcs_disconnect(c->ipcs); -- return rc; -- } -- } -- -- c->queue_backlog = queue_len; -- delay_next_flush(c, queue_len); -- -- } else { -- /* Event queue is empty, there is no backlog */ -- c->queue_backlog = 0; -- } -- -- return rc; --} -- --/*! -- * \internal -- * \brief Create an I/O vector for sending an IPC XML message -- * -- * \param[in] request Identifier for libqb response header -- * \param[in] message XML message to send -- * \param[in] max_send_size If 0, default IPC buffer size is used -- * \param[out] result Where to store prepared I/O vector -- * \param[out] bytes Size of prepared data in bytes -- * -- * \return Standard Pacemaker return code -- */ --int --pcmk__ipc_prepare_iov(uint32_t request, xmlNode *message, -- uint32_t max_send_size, struct iovec **result, -- ssize_t *bytes) --{ -- static unsigned int biggest = 0; -- struct iovec *iov; -- unsigned int total = 0; -- char *compressed = NULL; -- char *buffer = NULL; -- struct crm_ipc_response_header *header = NULL; -- -- if ((message == NULL) || (result == NULL)) { -- return EINVAL; -- } -- -- header = calloc(1, sizeof(struct crm_ipc_response_header)); -- if (header == NULL) { -- return ENOMEM; /* errno mightn't be set by allocator */ -- } -- -- buffer = dump_xml_unformatted(message); -- crm_ipc_init(); -- -- if (max_send_size == 0) { -- max_send_size = ipc_buffer_max; -- } -- CRM_LOG_ASSERT(max_send_size != 0); -- -- *result = NULL; -- iov = pcmk__new_ipc_event(); -- iov[0].iov_len = hdr_offset; -- iov[0].iov_base = header; -- -- header->version = PCMK_IPC_VERSION; -- header->size_uncompressed = 1 + strlen(buffer); -- total = iov[0].iov_len + header->size_uncompressed; -- -- if (total < max_send_size) { -- iov[1].iov_base = buffer; -- iov[1].iov_len = header->size_uncompressed; -- -- } else { -- unsigned int new_size = 0; -- -- if (pcmk__compress(buffer, (unsigned int) header->size_uncompressed, -- (unsigned int) max_send_size, &compressed, -- &new_size) == pcmk_rc_ok) { -- -- header->flags |= crm_ipc_compressed; -- header->size_compressed = new_size; -- -- iov[1].iov_len = header->size_compressed; -- iov[1].iov_base = compressed; -- -- free(buffer); -- -- biggest = QB_MAX(header->size_compressed, biggest); -- -- } else { -- crm_log_xml_trace(message, "EMSGSIZE"); -- biggest = QB_MAX(header->size_uncompressed, biggest); -- -- crm_err("Could not compress %u-byte message into less than IPC " -- "limit of %u bytes; set PCMK_ipc_buffer to higher value " -- "(%u bytes suggested)", -- header->size_uncompressed, max_send_size, 4 * biggest); -- -- free(compressed); -- free(buffer); -- pcmk_free_ipc_event(iov); -- return EMSGSIZE; -- } -- } -- -- header->qb.size = iov[0].iov_len + iov[1].iov_len; -- header->qb.id = (int32_t)request; /* Replying to a specific request */ -- -- *result = iov; -- CRM_ASSERT(header->qb.size > 0); -- if (bytes != NULL) { -- *bytes = header->qb.size; -- } -- return pcmk_rc_ok; --} -- --int --pcmk__ipc_send_iov(pcmk__client_t *c, struct iovec *iov, uint32_t flags) --{ -- int rc = pcmk_rc_ok; -- static uint32_t id = 1; -- struct crm_ipc_response_header *header = iov[0].iov_base; -- -- if (c->flags & pcmk__client_proxied) { -- /* _ALL_ replies to proxied connections need to be sent as events */ -- if (is_not_set(flags, crm_ipc_server_event)) { -- flags |= crm_ipc_server_event; -- /* this flag lets us know this was originally meant to be a response. -- * even though we're sending it over the event channel. */ -- flags |= crm_ipc_proxied_relay_response; -- } -- } -- -- header->flags |= flags; -- if (flags & crm_ipc_server_event) { -- header->qb.id = id++; /* We don't really use it, but doesn't hurt to set one */ -- -- if (flags & crm_ipc_server_free) { -- crm_trace("Sending the original to %p[%d]", c->ipcs, c->pid); -- add_event(c, iov); -- -- } else { -- struct iovec *iov_copy = pcmk__new_ipc_event(); -- -- crm_trace("Sending a copy to %p[%d]", c->ipcs, c->pid); -- iov_copy[0].iov_len = iov[0].iov_len; -- iov_copy[0].iov_base = malloc(iov[0].iov_len); -- memcpy(iov_copy[0].iov_base, iov[0].iov_base, iov[0].iov_len); -- -- iov_copy[1].iov_len = iov[1].iov_len; -- iov_copy[1].iov_base = malloc(iov[1].iov_len); -- memcpy(iov_copy[1].iov_base, iov[1].iov_base, iov[1].iov_len); -- -- add_event(c, iov_copy); -- } -- -- } else { -- ssize_t qb_rc; -- -- CRM_LOG_ASSERT(header->qb.id != 0); /* Replying to a specific request */ -- -- qb_rc = qb_ipcs_response_sendv(c->ipcs, iov, 2); -- if (qb_rc < header->qb.size) { -- if (qb_rc < 0) { -- rc = (int) -qb_rc; -- } -- crm_notice("Response %d to pid %d failed: %s " -- CRM_XS " bytes=%u rc=%lld ipcs=%p", -- header->qb.id, c->pid, pcmk_rc_str(rc), -- header->qb.size, (long long) qb_rc, c->ipcs); -- -- } else { -- crm_trace("Response %d sent, %lld bytes to %p[%d]", -- header->qb.id, (long long) qb_rc, c->ipcs, c->pid); -- } -- -- if (flags & crm_ipc_server_free) { -- pcmk_free_ipc_event(iov); -- } -- } -- -- if (flags & crm_ipc_server_event) { -- rc = crm_ipcs_flush_events(c); -- } else { -- crm_ipcs_flush_events(c); -- } -- -- if ((rc == EPIPE) || (rc == ENOTCONN)) { -- crm_trace("Client %p disconnected", c->ipcs); -- } -- return rc; --} -- --int --pcmk__ipc_send_xml(pcmk__client_t *c, uint32_t request, xmlNode *message, -- uint32_t flags) --{ -- struct iovec *iov = NULL; -- int rc = pcmk_rc_ok; -- -- if (c == NULL) { -- return EINVAL; -- } -- crm_ipc_init(); -- rc = pcmk__ipc_prepare_iov(request, message, ipc_buffer_max, &iov, NULL); -- if (rc == pcmk_rc_ok) { -- rc = pcmk__ipc_send_iov(c, iov, flags | crm_ipc_server_free); -- } else { -- pcmk_free_ipc_event(iov); -- crm_notice("IPC message to pid %d failed: %s " CRM_XS " rc=%d", -- c->pid, pcmk_rc_str(rc), rc); -- } -- return rc; --} -- --void --pcmk__ipc_send_ack_as(const char *function, int line, pcmk__client_t *c, -- uint32_t request, uint32_t flags, const char *tag) --{ -- if (flags & crm_ipc_client_response) { -- xmlNode *ack = create_xml_node(NULL, tag); -- -- crm_trace("Ack'ing IPC message from %s", pcmk__client_name(c)); -- c->request_id = 0; -- crm_xml_add(ack, "function", function); -- crm_xml_add_int(ack, "line", line); -- pcmk__ipc_send_xml(c, request, ack, flags); -- free_xml(ack); -- } --} -- --/*! -- * \internal -- * \brief Add an IPC server to the main loop for the pacemaker-based API -- * -- * \param[out] ipcs_ro New IPC server for read-only pacemaker-based API -- * \param[out] ipcs_rw New IPC server for read/write pacemaker-based API -- * \param[out] ipcs_shm New IPC server for shared-memory pacemaker-based API -- * \param[in] ro_cb IPC callbacks for read-only API -- * \param[in] rw_cb IPC callbacks for read/write and shared-memory APIs -- * -- * \note This function exits fatally if unable to create the servers. -- */ --void pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro, -- qb_ipcs_service_t **ipcs_rw, -- qb_ipcs_service_t **ipcs_shm, -- struct qb_ipcs_service_handlers *ro_cb, -- struct qb_ipcs_service_handlers *rw_cb) --{ -- *ipcs_ro = mainloop_add_ipc_server(PCMK__SERVER_BASED_RO, -- QB_IPC_NATIVE, ro_cb); -- -- *ipcs_rw = mainloop_add_ipc_server(PCMK__SERVER_BASED_RW, -- QB_IPC_NATIVE, rw_cb); -- -- *ipcs_shm = mainloop_add_ipc_server(PCMK__SERVER_BASED_SHM, -- QB_IPC_SHM, rw_cb); -- -- if (*ipcs_ro == NULL || *ipcs_rw == NULL || *ipcs_shm == NULL) { -- crm_err("Failed to create the CIB manager: exiting and inhibiting respawn"); -- crm_warn("Verify pacemaker and pacemaker_remote are not both enabled"); -- crm_exit(CRM_EX_FATAL); -- } --} -- --/*! -- * \internal -- * \brief Destroy IPC servers for pacemaker-based API -- * -- * \param[out] ipcs_ro IPC server for read-only pacemaker-based API -- * \param[out] ipcs_rw IPC server for read/write pacemaker-based API -- * \param[out] ipcs_shm IPC server for shared-memory pacemaker-based API -- * -- * \note This is a convenience function for calling qb_ipcs_destroy() for each -- * argument. -- */ --void --pcmk__stop_based_ipc(qb_ipcs_service_t *ipcs_ro, -- qb_ipcs_service_t *ipcs_rw, -- qb_ipcs_service_t *ipcs_shm) --{ -- qb_ipcs_destroy(ipcs_ro); -- qb_ipcs_destroy(ipcs_rw); -- qb_ipcs_destroy(ipcs_shm); --} -- --/*! -- * \internal -- * \brief Add an IPC server to the main loop for the pacemaker-controld API -- * -- * \param[in] cb IPC callbacks -- * -- * \return Newly created IPC server -- */ --qb_ipcs_service_t * --pcmk__serve_controld_ipc(struct qb_ipcs_service_handlers *cb) --{ -- return mainloop_add_ipc_server(CRM_SYSTEM_CRMD, QB_IPC_NATIVE, cb); --} -- --/*! -- * \internal -- * \brief Add an IPC server to the main loop for the pacemaker-attrd API -- * -- * \param[in] cb IPC callbacks -- * -- * \note This function exits fatally if unable to create the servers. -- */ --void --pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs, -- struct qb_ipcs_service_handlers *cb) --{ -- *ipcs = mainloop_add_ipc_server(T_ATTRD, QB_IPC_NATIVE, cb); -- -- if (*ipcs == NULL) { -- crm_err("Failed to create pacemaker-attrd server: exiting and inhibiting respawn"); -- crm_warn("Verify pacemaker and pacemaker_remote are not both enabled."); -- crm_exit(CRM_EX_FATAL); -- } --} -- --/*! -- * \internal -- * \brief Add an IPC server to the main loop for the pacemaker-fenced API -- * -- * \param[in] cb IPC callbacks -- * -- * \note This function exits fatally if unable to create the servers. -- */ --void --pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs, -- struct qb_ipcs_service_handlers *cb) --{ -- *ipcs = mainloop_add_ipc_server_with_prio("stonith-ng", QB_IPC_NATIVE, cb, -- QB_LOOP_HIGH); -- -- if (*ipcs == NULL) { -- crm_err("Failed to create fencer: exiting and inhibiting respawn."); -- crm_warn("Verify pacemaker and pacemaker_remote are not both enabled."); -- crm_exit(CRM_EX_FATAL); -- } --} -- --/* Client... */ -- --#define MIN_MSG_SIZE 12336 /* sizeof(struct qb_ipc_connection_response) */ --#define MAX_MSG_SIZE 128*1024 /* 128k default */ -- --struct crm_ipc_s { -- struct pollfd pfd; -- -- /* the max size we can send/receive over ipc */ -- unsigned int max_buf_size; -- /* Size of the allocated 'buffer' */ -- unsigned int buf_size; -- int msg_size; -- int need_reply; -- char *buffer; -- char *name; -- -- qb_ipcc_connection_t *ipc; -- --}; -- --static unsigned int --pick_ipc_buffer(unsigned int max) --{ -- static unsigned int global_max = 0; -- -- if (global_max == 0) { -- const char *env = getenv("PCMK_ipc_buffer"); -- -- if (env) { -- int env_max = crm_parse_int(env, "0"); -- -- global_max = (env_max > 0)? QB_MAX(MIN_MSG_SIZE, env_max) : MAX_MSG_SIZE; -- -- } else { -- global_max = MAX_MSG_SIZE; -- } -- } -- -- return QB_MAX(max, global_max); --} -- --crm_ipc_t * --crm_ipc_new(const char *name, size_t max_size) --{ -- crm_ipc_t *client = NULL; -- -- client = calloc(1, sizeof(crm_ipc_t)); -- -- client->name = strdup(name); -- client->buf_size = pick_ipc_buffer(max_size); -- client->buffer = malloc(client->buf_size); -- -- /* Clients initiating connection pick the max buf size */ -- client->max_buf_size = client->buf_size; -- -- client->pfd.fd = -1; -- client->pfd.events = POLLIN; -- client->pfd.revents = 0; -- -- return client; --} -- --/*! -- * \brief Establish an IPC connection to a Pacemaker component -- * -- * \param[in] client Connection instance obtained from crm_ipc_new() -- * -- * \return TRUE on success, FALSE otherwise (in which case errno will be set; -- * specifically, in case of discovering the remote side is not -- * authentic, its value is set to ECONNABORTED). -- */ --bool --crm_ipc_connect(crm_ipc_t * client) --{ -- uid_t cl_uid = 0; -- gid_t cl_gid = 0; -- pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; -- int rv; -- -- client->need_reply = FALSE; -- client->ipc = qb_ipcc_connect(client->name, client->buf_size); -- -- if (client->ipc == NULL) { -- crm_debug("Could not establish %s connection: %s (%d)", client->name, pcmk_strerror(errno), errno); -- return FALSE; -- } -- -- client->pfd.fd = crm_ipc_get_fd(client); -- if (client->pfd.fd < 0) { -- rv = errno; -- /* message already omitted */ -- crm_ipc_close(client); -- errno = rv; -- return FALSE; -- } -- -- rv = pcmk_daemon_user(&cl_uid, &cl_gid); -- if (rv < 0) { -- /* message already omitted */ -- crm_ipc_close(client); -- errno = -rv; -- return FALSE; -- } -- -- if (!(rv = crm_ipc_is_authentic_process(client->pfd.fd, cl_uid, cl_gid, -- &found_pid, &found_uid, -- &found_gid))) { -- crm_err("Daemon (IPC %s) is not authentic:" -- " process %lld (uid: %lld, gid: %lld)", -- client->name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -- (long long) found_uid, (long long) found_gid); -- crm_ipc_close(client); -- errno = ECONNABORTED; -- return FALSE; -- -- } else if (rv < 0) { -- errno = -rv; -- crm_perror(LOG_ERR, "Could not verify authenticity of daemon (IPC %s)", -- client->name); -- crm_ipc_close(client); -- errno = -rv; -- return FALSE; -- } -- -- qb_ipcc_context_set(client->ipc, client); -- --#ifdef HAVE_IPCS_GET_BUFFER_SIZE -- client->max_buf_size = qb_ipcc_get_buffer_size(client->ipc); -- if (client->max_buf_size > client->buf_size) { -- free(client->buffer); -- client->buffer = calloc(1, client->max_buf_size); -- client->buf_size = client->max_buf_size; -- } --#endif -- -- return TRUE; --} -- --void --crm_ipc_close(crm_ipc_t * client) --{ -- if (client) { -- crm_trace("Disconnecting %s IPC connection %p (%p)", client->name, client, client->ipc); -- -- if (client->ipc) { -- qb_ipcc_connection_t *ipc = client->ipc; -- -- client->ipc = NULL; -- qb_ipcc_disconnect(ipc); -- } -- } --} -- --void --crm_ipc_destroy(crm_ipc_t * client) --{ -- if (client) { -- if (client->ipc && qb_ipcc_is_connected(client->ipc)) { -- crm_notice("Destroying an active IPC connection to %s", client->name); -- /* The next line is basically unsafe -- * -- * If this connection was attached to mainloop and mainloop is active, -- * the 'disconnected' callback will end up back here and we'll end -- * up free'ing the memory twice - something that can still happen -- * even without this if we destroy a connection and it closes before -- * we call exit -- */ -- /* crm_ipc_close(client); */ -- } -- crm_trace("Destroying IPC connection to %s: %p", client->name, client); -- free(client->buffer); -- free(client->name); -- free(client); -- } --} -- --int --crm_ipc_get_fd(crm_ipc_t * client) --{ -- int fd = 0; -- -- if (client && client->ipc && (qb_ipcc_fd_get(client->ipc, &fd) == 0)) { -- return fd; -- } -- errno = EINVAL; -- crm_perror(LOG_ERR, "Could not obtain file IPC descriptor for %s", -- (client? client->name : "unspecified client")); -- return -errno; --} -- --bool --crm_ipc_connected(crm_ipc_t * client) --{ -- bool rc = FALSE; -- -- if (client == NULL) { -- crm_trace("No client"); -- return FALSE; -- -- } else if (client->ipc == NULL) { -- crm_trace("No connection"); -- return FALSE; -- -- } else if (client->pfd.fd < 0) { -- crm_trace("Bad descriptor"); -- return FALSE; -- } -- -- rc = qb_ipcc_is_connected(client->ipc); -- if (rc == FALSE) { -- client->pfd.fd = -EINVAL; -- } -- return rc; --} -- --/*! -- * \brief Check whether an IPC connection is ready to be read -- * -- * \param[in] client Connection to check -- * -- * \return Positive value if ready to be read, 0 if not ready, -errno on error -- */ --int --crm_ipc_ready(crm_ipc_t *client) --{ -- int rc; -- -- CRM_ASSERT(client != NULL); -- -- if (crm_ipc_connected(client) == FALSE) { -- return -ENOTCONN; -- } -- -- client->pfd.revents = 0; -- rc = poll(&(client->pfd), 1, 0); -- return (rc < 0)? -errno : rc; --} -- --// \return Standard Pacemaker return code --static int --crm_ipc_decompress(crm_ipc_t * client) --{ -- struct crm_ipc_response_header *header = (struct crm_ipc_response_header *)(void*)client->buffer; -- -- if (header->size_compressed) { -- int rc = 0; -- unsigned int size_u = 1 + header->size_uncompressed; -- /* never let buf size fall below our max size required for ipc reads. */ -- unsigned int new_buf_size = QB_MAX((hdr_offset + size_u), client->max_buf_size); -- char *uncompressed = calloc(1, new_buf_size); -- -- crm_trace("Decompressing message data %u bytes into %u bytes", -- header->size_compressed, size_u); -- -- rc = BZ2_bzBuffToBuffDecompress(uncompressed + hdr_offset, &size_u, -- client->buffer + hdr_offset, header->size_compressed, 1, 0); -- -- if (rc != BZ_OK) { -- crm_err("Decompression failed: %s " CRM_XS " bzerror=%d", -- bz2_strerror(rc), rc); -- free(uncompressed); -- return EILSEQ; -- } -- -- /* -- * This assert no longer holds true. For an identical msg, some clients may -- * require compression, and others may not. If that same msg (event) is sent -- * to multiple clients, it could result in some clients receiving a compressed -- * msg even though compression was not explicitly required for them. -- * -- * CRM_ASSERT((header->size_uncompressed + hdr_offset) >= ipc_buffer_max); -- */ -- CRM_ASSERT(size_u == header->size_uncompressed); -- -- memcpy(uncompressed, client->buffer, hdr_offset); /* Preserve the header */ -- header = (struct crm_ipc_response_header *)(void*)uncompressed; -- -- free(client->buffer); -- client->buf_size = new_buf_size; -- client->buffer = uncompressed; -- } -- -- CRM_ASSERT(client->buffer[hdr_offset + header->size_uncompressed - 1] == 0); -- return pcmk_rc_ok; --} -- --long --crm_ipc_read(crm_ipc_t * client) --{ -- struct crm_ipc_response_header *header = NULL; -- -- CRM_ASSERT(client != NULL); -- CRM_ASSERT(client->ipc != NULL); -- CRM_ASSERT(client->buffer != NULL); -- -- crm_ipc_init(); -- -- client->buffer[0] = 0; -- client->msg_size = qb_ipcc_event_recv(client->ipc, client->buffer, -- client->buf_size, 0); -- if (client->msg_size >= 0) { -- int rc = crm_ipc_decompress(client); -- -- if (rc != pcmk_rc_ok) { -- return pcmk_rc2legacy(rc); -- } -- -- header = (struct crm_ipc_response_header *)(void*)client->buffer; -- if(header->version > PCMK_IPC_VERSION) { -- crm_err("Filtering incompatible v%d IPC message, we only support versions <= %d", -- header->version, PCMK_IPC_VERSION); -- return -EBADMSG; -- } -- -- crm_trace("Received %s event %d, size=%u, rc=%d, text: %.100s", -- client->name, header->qb.id, header->qb.size, client->msg_size, -- client->buffer + hdr_offset); -- -- } else { -- crm_trace("No message from %s received: %s", client->name, pcmk_strerror(client->msg_size)); -- } -- -- if (crm_ipc_connected(client) == FALSE || client->msg_size == -ENOTCONN) { -- crm_err("Connection to %s failed", client->name); -- } -- -- if (header) { -- /* Data excluding the header */ -- return header->size_uncompressed; -- } -- return -ENOMSG; --} -- --const char * --crm_ipc_buffer(crm_ipc_t * client) --{ -- CRM_ASSERT(client != NULL); -- return client->buffer + sizeof(struct crm_ipc_response_header); --} -- --uint32_t --crm_ipc_buffer_flags(crm_ipc_t * client) --{ -- struct crm_ipc_response_header *header = NULL; -- -- CRM_ASSERT(client != NULL); -- if (client->buffer == NULL) { -- return 0; -- } -- -- header = (struct crm_ipc_response_header *)(void*)client->buffer; -- return header->flags; --} -- --const char * --crm_ipc_name(crm_ipc_t * client) --{ -- CRM_ASSERT(client != NULL); -- return client->name; --} -- --// \return Standard Pacemaker return code --static int --internal_ipc_get_reply(crm_ipc_t *client, int request_id, int ms_timeout, -- ssize_t *bytes) --{ -- time_t timeout = time(NULL) + 1 + (ms_timeout / 1000); -- int rc = pcmk_rc_ok; -- -- crm_ipc_init(); -- -- /* get the reply */ -- crm_trace("client %s waiting on reply to msg id %d", client->name, request_id); -- do { -- -- *bytes = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, 1000); -- if (*bytes > 0) { -- struct crm_ipc_response_header *hdr = NULL; -- -- rc = crm_ipc_decompress(client); -- if (rc != pcmk_rc_ok) { -- return rc; -- } -- -- hdr = (struct crm_ipc_response_header *)(void*)client->buffer; -- if (hdr->qb.id == request_id) { -- /* Got it */ -- break; -- } else if (hdr->qb.id < request_id) { -- xmlNode *bad = string2xml(crm_ipc_buffer(client)); -- -- crm_err("Discarding old reply %d (need %d)", hdr->qb.id, request_id); -- crm_log_xml_notice(bad, "OldIpcReply"); -- -- } else { -- xmlNode *bad = string2xml(crm_ipc_buffer(client)); -- -- crm_err("Discarding newer reply %d (need %d)", hdr->qb.id, request_id); -- crm_log_xml_notice(bad, "ImpossibleReply"); -- CRM_ASSERT(hdr->qb.id <= request_id); -- } -- } else if (crm_ipc_connected(client) == FALSE) { -- crm_err("Server disconnected client %s while waiting for msg id %d", client->name, -- request_id); -- break; -- } -- -- } while (time(NULL) < timeout); -- -- if (*bytes < 0) { -- rc = (int) -*bytes; // System errno -- } -- return rc; --} -- --/*! -- * \brief Send an IPC XML message -- * -- * \param[in] client Connection to IPC server -- * \param[in] message XML message to send -- * \param[in] flags Bitmask of crm_ipc_flags -- * \param[in] ms_timeout Give up if not sent within this much time -- * (5 seconds if 0, or no timeout if negative) -- * \param[out] reply Reply from server (or NULL if none) -- * -- * \return Negative errno on error, otherwise size of reply received in bytes -- * if reply was needed, otherwise number of bytes sent -- */ --int --crm_ipc_send(crm_ipc_t * client, xmlNode * message, enum crm_ipc_flags flags, int32_t ms_timeout, -- xmlNode ** reply) --{ -- int rc = 0; -- ssize_t qb_rc = 0; -- ssize_t bytes = 0; -- struct iovec *iov; -- static uint32_t id = 0; -- static int factor = 8; -- struct crm_ipc_response_header *header; -- -- crm_ipc_init(); -- -- if (client == NULL) { -- crm_notice("Can't send IPC request without connection (bug?): %.100s", -- message); -- return -ENOTCONN; -- -- } else if (crm_ipc_connected(client) == FALSE) { -- /* Don't even bother */ -- crm_notice("Can't send IPC request to %s: Connection closed", -- client->name); -- return -ENOTCONN; -- } -- -- if (ms_timeout == 0) { -- ms_timeout = 5000; -- } -- -- if (client->need_reply) { -- qb_rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, ms_timeout); -- if (qb_rc < 0) { -- crm_warn("Sending IPC to %s disabled until pending reply received", -- client->name); -- return -EALREADY; -- -- } else { -- crm_notice("Sending IPC to %s re-enabled after pending reply received", -- client->name); -- client->need_reply = FALSE; -- } -- } -- -- id++; -- CRM_LOG_ASSERT(id != 0); /* Crude wrap-around detection */ -- rc = pcmk__ipc_prepare_iov(id, message, client->max_buf_size, &iov, &bytes); -- if (rc != pcmk_rc_ok) { -- crm_warn("Couldn't prepare IPC request to %s: %s " CRM_XS " rc=%d", -- client->name, pcmk_rc_str(rc), rc); -- return pcmk_rc2legacy(rc); -- } -- -- header = iov[0].iov_base; -- header->flags |= flags; -- -- if(is_set(flags, crm_ipc_proxied)) { -- /* Don't look for a synchronous response */ -- clear_bit(flags, crm_ipc_client_response); -- } -- -- if(header->size_compressed) { -- if(factor < 10 && (client->max_buf_size / 10) < (bytes / factor)) { -- crm_notice("Compressed message exceeds %d0%% of configured IPC " -- "limit (%u bytes); consider setting PCMK_ipc_buffer to " -- "%u or higher", -- factor, client->max_buf_size, 2 * client->max_buf_size); -- factor++; -- } -- } -- -- crm_trace("Sending %s IPC request %d of %u bytes using %dms timeout", -- client->name, header->qb.id, header->qb.size, ms_timeout); -- -- if (ms_timeout > 0 || is_not_set(flags, crm_ipc_client_response)) { -- -- time_t timeout = time(NULL) + 1 + (ms_timeout / 1000); -- -- do { -- /* @TODO Is this check really needed? Won't qb_ipcc_sendv() return -- * an error if it's not connected? -- */ -- if (!crm_ipc_connected(client)) { -- goto send_cleanup; -- } -- -- qb_rc = qb_ipcc_sendv(client->ipc, iov, 2); -- } while ((qb_rc == -EAGAIN) && (time(NULL) < timeout)); -- -- rc = (int) qb_rc; // Negative of system errno, or bytes sent -- if (qb_rc <= 0) { -- goto send_cleanup; -- -- } else if (is_not_set(flags, crm_ipc_client_response)) { -- crm_trace("Not waiting for reply to %s IPC request %d", -- client->name, header->qb.id); -- goto send_cleanup; -- } -- -- rc = internal_ipc_get_reply(client, header->qb.id, ms_timeout, &bytes); -- if (rc != pcmk_rc_ok) { -- /* We didn't get the reply in time, so disable future sends for now. -- * The only alternative would be to close the connection since we -- * don't know how to detect and discard out-of-sequence replies. -- * -- * @TODO Implement out-of-sequence detection -- */ -- client->need_reply = TRUE; -- } -- rc = (int) bytes; // Negative system errno, or size of reply received -- -- } else { -- // No timeout, and client response needed -- do { -- qb_rc = qb_ipcc_sendv_recv(client->ipc, iov, 2, client->buffer, -- client->buf_size, -1); -- } while ((qb_rc == -EAGAIN) && crm_ipc_connected(client)); -- rc = (int) qb_rc; // Negative system errno, or size of reply received -- } -- -- if (rc > 0) { -- struct crm_ipc_response_header *hdr = (struct crm_ipc_response_header *)(void*)client->buffer; -- -- crm_trace("Received %d-byte reply %d to %s IPC %d: %.100s", -- rc, hdr->qb.id, client->name, header->qb.id, -- crm_ipc_buffer(client)); -- -- if (reply) { -- *reply = string2xml(crm_ipc_buffer(client)); -- } -- -- } else { -- crm_trace("No reply to %s IPC %d: rc=%d", -- client->name, header->qb.id, rc); -- } -- -- send_cleanup: -- if (crm_ipc_connected(client) == FALSE) { -- crm_notice("Couldn't send %s IPC request %d: Connection closed " -- CRM_XS " rc=%d", client->name, header->qb.id, rc); -- -- } else if (rc == -ETIMEDOUT) { -- crm_warn("%s IPC request %d failed: %s after %dms " CRM_XS " rc=%d", -- client->name, header->qb.id, pcmk_strerror(rc), ms_timeout, -- rc); -- crm_write_blackbox(0, NULL); -- -- } else if (rc <= 0) { -- crm_warn("%s IPC request %d failed: %s " CRM_XS " rc=%d", -- client->name, header->qb.id, -- ((rc == 0)? "No bytes sent" : pcmk_strerror(rc)), rc); -- } -- -- pcmk_free_ipc_event(iov); -- return rc; --} -- --int --crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid, -- pid_t *gotpid, uid_t *gotuid, gid_t *gotgid) { -- int ret = 0; -- pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; --#if defined(US_AUTH_PEERCRED_UCRED) -- struct ucred ucred; -- socklen_t ucred_len = sizeof(ucred); -- -- if (!getsockopt(sock, SOL_SOCKET, SO_PEERCRED, -- &ucred, &ucred_len) -- && ucred_len == sizeof(ucred)) { -- found_pid = ucred.pid; found_uid = ucred.uid; found_gid = ucred.gid; -- --#elif defined(US_AUTH_PEERCRED_SOCKPEERCRED) -- struct sockpeercred sockpeercred; -- socklen_t sockpeercred_len = sizeof(sockpeercred); -- -- if (!getsockopt(sock, SOL_SOCKET, SO_PEERCRED, -- &sockpeercred, &sockpeercred_len) -- && sockpeercred_len == sizeof(sockpeercred_len)) { -- found_pid = sockpeercred.pid; -- found_uid = sockpeercred.uid; found_gid = sockpeercred.gid; -- --#elif defined(US_AUTH_GETPEEREID) -- if (!getpeereid(sock, &found_uid, &found_gid)) { -- found_pid = PCMK__SPECIAL_PID; /* cannot obtain PID (FreeBSD) */ -- --#elif defined(US_AUTH_GETPEERUCRED) -- ucred_t *ucred; -- if (!getpeerucred(sock, &ucred)) { -- errno = 0; -- found_pid = ucred_getpid(ucred); -- found_uid = ucred_geteuid(ucred); found_gid = ucred_getegid(ucred); -- ret = -errno; -- ucred_free(ucred); -- if (ret) { -- return (ret < 0) ? ret : -pcmk_err_generic; -- } -- --#else --# error "No way to authenticate a Unix socket peer" -- errno = 0; -- if (0) { --#endif -- if (gotpid != NULL) { -- *gotpid = found_pid; -- } -- if (gotuid != NULL) { -- *gotuid = found_uid; -- } -- if (gotgid != NULL) { -- *gotgid = found_gid; -- } -- ret = (found_uid == 0 || found_uid == refuid || found_gid == refgid); -- } else { -- ret = (errno > 0) ? -errno : -pcmk_err_generic; -- } -- -- return ret; --} -- --int --pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid, -- gid_t refgid, pid_t *gotpid) --{ -- static char last_asked_name[PATH_MAX / 2] = ""; /* log spam prevention */ -- int fd; -- int rc = pcmk_rc_ipc_unresponsive; -- int auth_rc = 0; -- int32_t qb_rc; -- pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; -- qb_ipcc_connection_t *c; -- -- c = qb_ipcc_connect(name, 0); -- if (c == NULL) { -- crm_info("Could not connect to %s IPC: %s", name, strerror(errno)); -- rc = pcmk_rc_ipc_unresponsive; -- goto bail; -- } -- -- qb_rc = qb_ipcc_fd_get(c, &fd); -- if (qb_rc != 0) { -- rc = (int) -qb_rc; // System errno -- crm_err("Could not get fd from %s IPC: %s " CRM_XS " rc=%d", -- name, pcmk_rc_str(rc), rc); -- goto bail; -- } -- -- auth_rc = crm_ipc_is_authentic_process(fd, refuid, refgid, &found_pid, -- &found_uid, &found_gid); -- if (auth_rc < 0) { -- rc = pcmk_legacy2rc(auth_rc); -- crm_err("Could not get peer credentials from %s IPC: %s " -- CRM_XS " rc=%d", name, pcmk_rc_str(rc), rc); -- goto bail; -- } -- -- if (gotpid != NULL) { -- *gotpid = found_pid; -- } -- -- if (auth_rc == 0) { -- crm_err("Daemon (IPC %s) effectively blocked with unauthorized" -- " process %lld (uid: %lld, gid: %lld)", -- name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -- (long long) found_uid, (long long) found_gid); -- rc = pcmk_rc_ipc_unauthorized; -- goto bail; -- } -- -- rc = pcmk_rc_ok; -- if ((found_uid != refuid || found_gid != refgid) -- && strncmp(last_asked_name, name, sizeof(last_asked_name))) { -- if ((found_uid == 0) && (refuid != 0)) { -- crm_warn("Daemon (IPC %s) runs as root, whereas the expected" -- " credentials are %lld:%lld, hazard of violating" -- " the least privilege principle", -- name, (long long) refuid, (long long) refgid); -- } else { -- crm_notice("Daemon (IPC %s) runs as %lld:%lld, whereas the" -- " expected credentials are %lld:%lld, which may" -- " mean a different set of privileges than expected", -- name, (long long) found_uid, (long long) found_gid, -- (long long) refuid, (long long) refgid); -- } -- memccpy(last_asked_name, name, '\0', sizeof(last_asked_name)); -- } -- --bail: -- if (c != NULL) { -- qb_ipcc_disconnect(c); -- } -- return rc; --} -- -- --/* Utils */ -- --xmlNode * --create_hello_message(const char *uuid, -- const char *client_name, const char *major_version, const char *minor_version) --{ -- xmlNode *hello_node = NULL; -- xmlNode *hello = NULL; -- -- if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name) -- || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) { -- crm_err("Could not create IPC hello message from %s (UUID %s): " -- "missing information", -- client_name? client_name : "unknown client", -- uuid? uuid : "unknown"); -- return NULL; -- } -- -- hello_node = create_xml_node(NULL, XML_TAG_OPTIONS); -- if (hello_node == NULL) { -- crm_err("Could not create IPC hello message from %s (UUID %s): " -- "Message data creation failed", client_name, uuid); -- return NULL; -- } -- -- crm_xml_add(hello_node, "major_version", major_version); -- crm_xml_add(hello_node, "minor_version", minor_version); -- crm_xml_add(hello_node, "client_name", client_name); -- crm_xml_add(hello_node, "client_uuid", uuid); -- -- hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid); -- if (hello == NULL) { -- crm_err("Could not create IPC hello message from %s (UUID %s): " -- "Request creation failed", client_name, uuid); -- return NULL; -- } -- free_xml(hello_node); -- -- crm_trace("Created hello message from %s (UUID %s)", client_name, uuid); -- return hello; --} -diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c -new file mode 100644 -index 0000000..7737588 ---- /dev/null -+++ b/lib/common/ipc_client.c -@@ -0,0 +1,755 @@ -+/* -+ * Copyright 2004-2020 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. -+ * -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -+ */ -+ -+#include -+ -+#if defined(US_AUTH_PEERCRED_UCRED) || defined(US_AUTH_PEERCRED_SOCKPEERCRED) -+# ifdef US_AUTH_PEERCRED_UCRED -+# ifndef _GNU_SOURCE -+# define _GNU_SOURCE -+# endif -+# endif -+# include -+#elif defined(US_AUTH_GETPEERUCRED) -+# include -+#endif -+ -+#include -+#include -+#include -+#include -+ -+#include /* indirectly: pcmk_err_generic */ -+#include -+#include -+#include -+#include "crmcommon_private.h" -+ -+struct crm_ipc_s { -+ struct pollfd pfd; -+ unsigned int max_buf_size; // maximum bytes we can send or receive over IPC -+ unsigned int buf_size; // size of allocated buffer -+ int msg_size; -+ int need_reply; -+ char *buffer; -+ char *name; -+ qb_ipcc_connection_t *ipc; -+}; -+ -+crm_ipc_t * -+crm_ipc_new(const char *name, size_t max_size) -+{ -+ crm_ipc_t *client = NULL; -+ -+ client = calloc(1, sizeof(crm_ipc_t)); -+ -+ client->name = strdup(name); -+ client->buf_size = pcmk__ipc_buffer_size(max_size); -+ client->buffer = malloc(client->buf_size); -+ -+ /* Clients initiating connection pick the max buf size */ -+ client->max_buf_size = client->buf_size; -+ -+ client->pfd.fd = -1; -+ client->pfd.events = POLLIN; -+ client->pfd.revents = 0; -+ -+ return client; -+} -+ -+/*! -+ * \brief Establish an IPC connection to a Pacemaker component -+ * -+ * \param[in] client Connection instance obtained from crm_ipc_new() -+ * -+ * \return TRUE on success, FALSE otherwise (in which case errno will be set; -+ * specifically, in case of discovering the remote side is not -+ * authentic, its value is set to ECONNABORTED). -+ */ -+bool -+crm_ipc_connect(crm_ipc_t * client) -+{ -+ uid_t cl_uid = 0; -+ gid_t cl_gid = 0; -+ pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; -+ int rv; -+ -+ client->need_reply = FALSE; -+ client->ipc = qb_ipcc_connect(client->name, client->buf_size); -+ -+ if (client->ipc == NULL) { -+ crm_debug("Could not establish %s connection: %s (%d)", client->name, pcmk_strerror(errno), errno); -+ return FALSE; -+ } -+ -+ client->pfd.fd = crm_ipc_get_fd(client); -+ if (client->pfd.fd < 0) { -+ rv = errno; -+ /* message already omitted */ -+ crm_ipc_close(client); -+ errno = rv; -+ return FALSE; -+ } -+ -+ rv = pcmk_daemon_user(&cl_uid, &cl_gid); -+ if (rv < 0) { -+ /* message already omitted */ -+ crm_ipc_close(client); -+ errno = -rv; -+ return FALSE; -+ } -+ -+ if (!(rv = crm_ipc_is_authentic_process(client->pfd.fd, cl_uid, cl_gid, -+ &found_pid, &found_uid, -+ &found_gid))) { -+ crm_err("Daemon (IPC %s) is not authentic:" -+ " process %lld (uid: %lld, gid: %lld)", -+ client->name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ crm_ipc_close(client); -+ errno = ECONNABORTED; -+ return FALSE; -+ -+ } else if (rv < 0) { -+ errno = -rv; -+ crm_perror(LOG_ERR, "Could not verify authenticity of daemon (IPC %s)", -+ client->name); -+ crm_ipc_close(client); -+ errno = -rv; -+ return FALSE; -+ } -+ -+ qb_ipcc_context_set(client->ipc, client); -+ -+#ifdef HAVE_IPCS_GET_BUFFER_SIZE -+ client->max_buf_size = qb_ipcc_get_buffer_size(client->ipc); -+ if (client->max_buf_size > client->buf_size) { -+ free(client->buffer); -+ client->buffer = calloc(1, client->max_buf_size); -+ client->buf_size = client->max_buf_size; -+ } -+#endif -+ -+ return TRUE; -+} -+ -+void -+crm_ipc_close(crm_ipc_t * client) -+{ -+ if (client) { -+ crm_trace("Disconnecting %s IPC connection %p (%p)", client->name, client, client->ipc); -+ -+ if (client->ipc) { -+ qb_ipcc_connection_t *ipc = client->ipc; -+ -+ client->ipc = NULL; -+ qb_ipcc_disconnect(ipc); -+ } -+ } -+} -+ -+void -+crm_ipc_destroy(crm_ipc_t * client) -+{ -+ if (client) { -+ if (client->ipc && qb_ipcc_is_connected(client->ipc)) { -+ crm_notice("Destroying an active IPC connection to %s", client->name); -+ /* The next line is basically unsafe -+ * -+ * If this connection was attached to mainloop and mainloop is active, -+ * the 'disconnected' callback will end up back here and we'll end -+ * up free'ing the memory twice - something that can still happen -+ * even without this if we destroy a connection and it closes before -+ * we call exit -+ */ -+ /* crm_ipc_close(client); */ -+ } -+ crm_trace("Destroying IPC connection to %s: %p", client->name, client); -+ free(client->buffer); -+ free(client->name); -+ free(client); -+ } -+} -+ -+int -+crm_ipc_get_fd(crm_ipc_t * client) -+{ -+ int fd = 0; -+ -+ if (client && client->ipc && (qb_ipcc_fd_get(client->ipc, &fd) == 0)) { -+ return fd; -+ } -+ errno = EINVAL; -+ crm_perror(LOG_ERR, "Could not obtain file IPC descriptor for %s", -+ (client? client->name : "unspecified client")); -+ return -errno; -+} -+ -+bool -+crm_ipc_connected(crm_ipc_t * client) -+{ -+ bool rc = FALSE; -+ -+ if (client == NULL) { -+ crm_trace("No client"); -+ return FALSE; -+ -+ } else if (client->ipc == NULL) { -+ crm_trace("No connection"); -+ return FALSE; -+ -+ } else if (client->pfd.fd < 0) { -+ crm_trace("Bad descriptor"); -+ return FALSE; -+ } -+ -+ rc = qb_ipcc_is_connected(client->ipc); -+ if (rc == FALSE) { -+ client->pfd.fd = -EINVAL; -+ } -+ return rc; -+} -+ -+/*! -+ * \brief Check whether an IPC connection is ready to be read -+ * -+ * \param[in] client Connection to check -+ * -+ * \return Positive value if ready to be read, 0 if not ready, -errno on error -+ */ -+int -+crm_ipc_ready(crm_ipc_t *client) -+{ -+ int rc; -+ -+ CRM_ASSERT(client != NULL); -+ -+ if (crm_ipc_connected(client) == FALSE) { -+ return -ENOTCONN; -+ } -+ -+ client->pfd.revents = 0; -+ rc = poll(&(client->pfd), 1, 0); -+ return (rc < 0)? -errno : rc; -+} -+ -+// \return Standard Pacemaker return code -+static int -+crm_ipc_decompress(crm_ipc_t * client) -+{ -+ pcmk__ipc_header_t *header = (pcmk__ipc_header_t *)(void*)client->buffer; -+ -+ if (header->size_compressed) { -+ int rc = 0; -+ unsigned int size_u = 1 + header->size_uncompressed; -+ /* never let buf size fall below our max size required for ipc reads. */ -+ unsigned int new_buf_size = QB_MAX((sizeof(pcmk__ipc_header_t) + size_u), client->max_buf_size); -+ char *uncompressed = calloc(1, new_buf_size); -+ -+ crm_trace("Decompressing message data %u bytes into %u bytes", -+ header->size_compressed, size_u); -+ -+ rc = BZ2_bzBuffToBuffDecompress(uncompressed + sizeof(pcmk__ipc_header_t), &size_u, -+ client->buffer + sizeof(pcmk__ipc_header_t), header->size_compressed, 1, 0); -+ -+ if (rc != BZ_OK) { -+ crm_err("Decompression failed: %s " CRM_XS " bzerror=%d", -+ bz2_strerror(rc), rc); -+ free(uncompressed); -+ return EILSEQ; -+ } -+ -+ /* -+ * This assert no longer holds true. For an identical msg, some clients may -+ * require compression, and others may not. If that same msg (event) is sent -+ * to multiple clients, it could result in some clients receiving a compressed -+ * msg even though compression was not explicitly required for them. -+ * -+ * CRM_ASSERT((header->size_uncompressed + sizeof(pcmk__ipc_header_t)) >= ipc_buffer_max); -+ */ -+ CRM_ASSERT(size_u == header->size_uncompressed); -+ -+ memcpy(uncompressed, client->buffer, sizeof(pcmk__ipc_header_t)); /* Preserve the header */ -+ header = (pcmk__ipc_header_t *)(void*)uncompressed; -+ -+ free(client->buffer); -+ client->buf_size = new_buf_size; -+ client->buffer = uncompressed; -+ } -+ -+ CRM_ASSERT(client->buffer[sizeof(pcmk__ipc_header_t) + header->size_uncompressed - 1] == 0); -+ return pcmk_rc_ok; -+} -+ -+long -+crm_ipc_read(crm_ipc_t * client) -+{ -+ pcmk__ipc_header_t *header = NULL; -+ -+ CRM_ASSERT(client != NULL); -+ CRM_ASSERT(client->ipc != NULL); -+ CRM_ASSERT(client->buffer != NULL); -+ -+ client->buffer[0] = 0; -+ client->msg_size = qb_ipcc_event_recv(client->ipc, client->buffer, -+ client->buf_size, 0); -+ if (client->msg_size >= 0) { -+ int rc = crm_ipc_decompress(client); -+ -+ if (rc != pcmk_rc_ok) { -+ return pcmk_rc2legacy(rc); -+ } -+ -+ header = (pcmk__ipc_header_t *)(void*)client->buffer; -+ if (!pcmk__valid_ipc_header(header)) { -+ return -EBADMSG; -+ } -+ -+ crm_trace("Received %s event %d, size=%u, rc=%d, text: %.100s", -+ client->name, header->qb.id, header->qb.size, client->msg_size, -+ client->buffer + sizeof(pcmk__ipc_header_t)); -+ -+ } else { -+ crm_trace("No message from %s received: %s", client->name, pcmk_strerror(client->msg_size)); -+ } -+ -+ if (crm_ipc_connected(client) == FALSE || client->msg_size == -ENOTCONN) { -+ crm_err("Connection to %s failed", client->name); -+ } -+ -+ if (header) { -+ /* Data excluding the header */ -+ return header->size_uncompressed; -+ } -+ return -ENOMSG; -+} -+ -+const char * -+crm_ipc_buffer(crm_ipc_t * client) -+{ -+ CRM_ASSERT(client != NULL); -+ return client->buffer + sizeof(pcmk__ipc_header_t); -+} -+ -+uint32_t -+crm_ipc_buffer_flags(crm_ipc_t * client) -+{ -+ pcmk__ipc_header_t *header = NULL; -+ -+ CRM_ASSERT(client != NULL); -+ if (client->buffer == NULL) { -+ return 0; -+ } -+ -+ header = (pcmk__ipc_header_t *)(void*)client->buffer; -+ return header->flags; -+} -+ -+const char * -+crm_ipc_name(crm_ipc_t * client) -+{ -+ CRM_ASSERT(client != NULL); -+ return client->name; -+} -+ -+// \return Standard Pacemaker return code -+static int -+internal_ipc_get_reply(crm_ipc_t *client, int request_id, int ms_timeout, -+ ssize_t *bytes) -+{ -+ time_t timeout = time(NULL) + 1 + (ms_timeout / 1000); -+ int rc = pcmk_rc_ok; -+ -+ /* get the reply */ -+ crm_trace("client %s waiting on reply to msg id %d", client->name, request_id); -+ do { -+ -+ *bytes = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, 1000); -+ if (*bytes > 0) { -+ pcmk__ipc_header_t *hdr = NULL; -+ -+ rc = crm_ipc_decompress(client); -+ if (rc != pcmk_rc_ok) { -+ return rc; -+ } -+ -+ hdr = (pcmk__ipc_header_t *)(void*)client->buffer; -+ if (hdr->qb.id == request_id) { -+ /* Got it */ -+ break; -+ } else if (hdr->qb.id < request_id) { -+ xmlNode *bad = string2xml(crm_ipc_buffer(client)); -+ -+ crm_err("Discarding old reply %d (need %d)", hdr->qb.id, request_id); -+ crm_log_xml_notice(bad, "OldIpcReply"); -+ -+ } else { -+ xmlNode *bad = string2xml(crm_ipc_buffer(client)); -+ -+ crm_err("Discarding newer reply %d (need %d)", hdr->qb.id, request_id); -+ crm_log_xml_notice(bad, "ImpossibleReply"); -+ CRM_ASSERT(hdr->qb.id <= request_id); -+ } -+ } else if (crm_ipc_connected(client) == FALSE) { -+ crm_err("Server disconnected client %s while waiting for msg id %d", client->name, -+ request_id); -+ break; -+ } -+ -+ } while (time(NULL) < timeout); -+ -+ if (*bytes < 0) { -+ rc = (int) -*bytes; // System errno -+ } -+ return rc; -+} -+ -+/*! -+ * \brief Send an IPC XML message -+ * -+ * \param[in] client Connection to IPC server -+ * \param[in] message XML message to send -+ * \param[in] flags Bitmask of crm_ipc_flags -+ * \param[in] ms_timeout Give up if not sent within this much time -+ * (5 seconds if 0, or no timeout if negative) -+ * \param[out] reply Reply from server (or NULL if none) -+ * -+ * \return Negative errno on error, otherwise size of reply received in bytes -+ * if reply was needed, otherwise number of bytes sent -+ */ -+int -+crm_ipc_send(crm_ipc_t * client, xmlNode * message, enum crm_ipc_flags flags, int32_t ms_timeout, -+ xmlNode ** reply) -+{ -+ int rc = 0; -+ ssize_t qb_rc = 0; -+ ssize_t bytes = 0; -+ struct iovec *iov; -+ static uint32_t id = 0; -+ static int factor = 8; -+ pcmk__ipc_header_t *header; -+ -+ if (client == NULL) { -+ crm_notice("Can't send IPC request without connection (bug?): %.100s", -+ message); -+ return -ENOTCONN; -+ -+ } else if (crm_ipc_connected(client) == FALSE) { -+ /* Don't even bother */ -+ crm_notice("Can't send IPC request to %s: Connection closed", -+ client->name); -+ return -ENOTCONN; -+ } -+ -+ if (ms_timeout == 0) { -+ ms_timeout = 5000; -+ } -+ -+ if (client->need_reply) { -+ qb_rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, ms_timeout); -+ if (qb_rc < 0) { -+ crm_warn("Sending IPC to %s disabled until pending reply received", -+ client->name); -+ return -EALREADY; -+ -+ } else { -+ crm_notice("Sending IPC to %s re-enabled after pending reply received", -+ client->name); -+ client->need_reply = FALSE; -+ } -+ } -+ -+ id++; -+ CRM_LOG_ASSERT(id != 0); /* Crude wrap-around detection */ -+ rc = pcmk__ipc_prepare_iov(id, message, client->max_buf_size, &iov, &bytes); -+ if (rc != pcmk_rc_ok) { -+ crm_warn("Couldn't prepare IPC request to %s: %s " CRM_XS " rc=%d", -+ client->name, pcmk_rc_str(rc), rc); -+ return pcmk_rc2legacy(rc); -+ } -+ -+ header = iov[0].iov_base; -+ header->flags |= flags; -+ -+ if(is_set(flags, crm_ipc_proxied)) { -+ /* Don't look for a synchronous response */ -+ clear_bit(flags, crm_ipc_client_response); -+ } -+ -+ if(header->size_compressed) { -+ if(factor < 10 && (client->max_buf_size / 10) < (bytes / factor)) { -+ crm_notice("Compressed message exceeds %d0%% of configured IPC " -+ "limit (%u bytes); consider setting PCMK_ipc_buffer to " -+ "%u or higher", -+ factor, client->max_buf_size, 2 * client->max_buf_size); -+ factor++; -+ } -+ } -+ -+ crm_trace("Sending %s IPC request %d of %u bytes using %dms timeout", -+ client->name, header->qb.id, header->qb.size, ms_timeout); -+ -+ if (ms_timeout > 0 || is_not_set(flags, crm_ipc_client_response)) { -+ -+ time_t timeout = time(NULL) + 1 + (ms_timeout / 1000); -+ -+ do { -+ /* @TODO Is this check really needed? Won't qb_ipcc_sendv() return -+ * an error if it's not connected? -+ */ -+ if (!crm_ipc_connected(client)) { -+ goto send_cleanup; -+ } -+ -+ qb_rc = qb_ipcc_sendv(client->ipc, iov, 2); -+ } while ((qb_rc == -EAGAIN) && (time(NULL) < timeout)); -+ -+ rc = (int) qb_rc; // Negative of system errno, or bytes sent -+ if (qb_rc <= 0) { -+ goto send_cleanup; -+ -+ } else if (is_not_set(flags, crm_ipc_client_response)) { -+ crm_trace("Not waiting for reply to %s IPC request %d", -+ client->name, header->qb.id); -+ goto send_cleanup; -+ } -+ -+ rc = internal_ipc_get_reply(client, header->qb.id, ms_timeout, &bytes); -+ if (rc != pcmk_rc_ok) { -+ /* We didn't get the reply in time, so disable future sends for now. -+ * The only alternative would be to close the connection since we -+ * don't know how to detect and discard out-of-sequence replies. -+ * -+ * @TODO Implement out-of-sequence detection -+ */ -+ client->need_reply = TRUE; -+ } -+ rc = (int) bytes; // Negative system errno, or size of reply received -+ -+ } else { -+ // No timeout, and client response needed -+ do { -+ qb_rc = qb_ipcc_sendv_recv(client->ipc, iov, 2, client->buffer, -+ client->buf_size, -1); -+ } while ((qb_rc == -EAGAIN) && crm_ipc_connected(client)); -+ rc = (int) qb_rc; // Negative system errno, or size of reply received -+ } -+ -+ if (rc > 0) { -+ pcmk__ipc_header_t *hdr = (pcmk__ipc_header_t *)(void*)client->buffer; -+ -+ crm_trace("Received %d-byte reply %d to %s IPC %d: %.100s", -+ rc, hdr->qb.id, client->name, header->qb.id, -+ crm_ipc_buffer(client)); -+ -+ if (reply) { -+ *reply = string2xml(crm_ipc_buffer(client)); -+ } -+ -+ } else { -+ crm_trace("No reply to %s IPC %d: rc=%d", -+ client->name, header->qb.id, rc); -+ } -+ -+ send_cleanup: -+ if (crm_ipc_connected(client) == FALSE) { -+ crm_notice("Couldn't send %s IPC request %d: Connection closed " -+ CRM_XS " rc=%d", client->name, header->qb.id, rc); -+ -+ } else if (rc == -ETIMEDOUT) { -+ crm_warn("%s IPC request %d failed: %s after %dms " CRM_XS " rc=%d", -+ client->name, header->qb.id, pcmk_strerror(rc), ms_timeout, -+ rc); -+ crm_write_blackbox(0, NULL); -+ -+ } else if (rc <= 0) { -+ crm_warn("%s IPC request %d failed: %s " CRM_XS " rc=%d", -+ client->name, header->qb.id, -+ ((rc == 0)? "No bytes sent" : pcmk_strerror(rc)), rc); -+ } -+ -+ pcmk_free_ipc_event(iov); -+ return rc; -+} -+ -+int -+crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid, -+ pid_t *gotpid, uid_t *gotuid, gid_t *gotgid) { -+ int ret = 0; -+ pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; -+#if defined(US_AUTH_PEERCRED_UCRED) -+ struct ucred ucred; -+ socklen_t ucred_len = sizeof(ucred); -+ -+ if (!getsockopt(sock, SOL_SOCKET, SO_PEERCRED, -+ &ucred, &ucred_len) -+ && ucred_len == sizeof(ucred)) { -+ found_pid = ucred.pid; found_uid = ucred.uid; found_gid = ucred.gid; -+ -+#elif defined(US_AUTH_PEERCRED_SOCKPEERCRED) -+ struct sockpeercred sockpeercred; -+ socklen_t sockpeercred_len = sizeof(sockpeercred); -+ -+ if (!getsockopt(sock, SOL_SOCKET, SO_PEERCRED, -+ &sockpeercred, &sockpeercred_len) -+ && sockpeercred_len == sizeof(sockpeercred_len)) { -+ found_pid = sockpeercred.pid; -+ found_uid = sockpeercred.uid; found_gid = sockpeercred.gid; -+ -+#elif defined(US_AUTH_GETPEEREID) -+ if (!getpeereid(sock, &found_uid, &found_gid)) { -+ found_pid = PCMK__SPECIAL_PID; /* cannot obtain PID (FreeBSD) */ -+ -+#elif defined(US_AUTH_GETPEERUCRED) -+ ucred_t *ucred; -+ if (!getpeerucred(sock, &ucred)) { -+ errno = 0; -+ found_pid = ucred_getpid(ucred); -+ found_uid = ucred_geteuid(ucred); found_gid = ucred_getegid(ucred); -+ ret = -errno; -+ ucred_free(ucred); -+ if (ret) { -+ return (ret < 0) ? ret : -pcmk_err_generic; -+ } -+ -+#else -+# error "No way to authenticate a Unix socket peer" -+ errno = 0; -+ if (0) { -+#endif -+ if (gotpid != NULL) { -+ *gotpid = found_pid; -+ } -+ if (gotuid != NULL) { -+ *gotuid = found_uid; -+ } -+ if (gotgid != NULL) { -+ *gotgid = found_gid; -+ } -+ ret = (found_uid == 0 || found_uid == refuid || found_gid == refgid); -+ } else { -+ ret = (errno > 0) ? -errno : -pcmk_err_generic; -+ } -+ -+ return ret; -+} -+ -+int -+pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid, -+ gid_t refgid, pid_t *gotpid) -+{ -+ static char last_asked_name[PATH_MAX / 2] = ""; /* log spam prevention */ -+ int fd; -+ int rc = pcmk_rc_ipc_unresponsive; -+ int auth_rc = 0; -+ int32_t qb_rc; -+ pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; -+ qb_ipcc_connection_t *c; -+ -+ c = qb_ipcc_connect(name, 0); -+ if (c == NULL) { -+ crm_info("Could not connect to %s IPC: %s", name, strerror(errno)); -+ rc = pcmk_rc_ipc_unresponsive; -+ goto bail; -+ } -+ -+ qb_rc = qb_ipcc_fd_get(c, &fd); -+ if (qb_rc != 0) { -+ rc = (int) -qb_rc; // System errno -+ crm_err("Could not get fd from %s IPC: %s " CRM_XS " rc=%d", -+ name, pcmk_rc_str(rc), rc); -+ goto bail; -+ } -+ -+ auth_rc = crm_ipc_is_authentic_process(fd, refuid, refgid, &found_pid, -+ &found_uid, &found_gid); -+ if (auth_rc < 0) { -+ rc = pcmk_legacy2rc(auth_rc); -+ crm_err("Could not get peer credentials from %s IPC: %s " -+ CRM_XS " rc=%d", name, pcmk_rc_str(rc), rc); -+ goto bail; -+ } -+ -+ if (gotpid != NULL) { -+ *gotpid = found_pid; -+ } -+ -+ if (auth_rc == 0) { -+ crm_err("Daemon (IPC %s) effectively blocked with unauthorized" -+ " process %lld (uid: %lld, gid: %lld)", -+ name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); -+ rc = pcmk_rc_ipc_unauthorized; -+ goto bail; -+ } -+ -+ rc = pcmk_rc_ok; -+ if ((found_uid != refuid || found_gid != refgid) -+ && strncmp(last_asked_name, name, sizeof(last_asked_name))) { -+ if ((found_uid == 0) && (refuid != 0)) { -+ crm_warn("Daemon (IPC %s) runs as root, whereas the expected" -+ " credentials are %lld:%lld, hazard of violating" -+ " the least privilege principle", -+ name, (long long) refuid, (long long) refgid); -+ } else { -+ crm_notice("Daemon (IPC %s) runs as %lld:%lld, whereas the" -+ " expected credentials are %lld:%lld, which may" -+ " mean a different set of privileges than expected", -+ name, (long long) found_uid, (long long) found_gid, -+ (long long) refuid, (long long) refgid); -+ } -+ memccpy(last_asked_name, name, '\0', sizeof(last_asked_name)); -+ } -+ -+bail: -+ if (c != NULL) { -+ qb_ipcc_disconnect(c); -+ } -+ return rc; -+} -+ -+xmlNode * -+create_hello_message(const char *uuid, -+ const char *client_name, const char *major_version, const char *minor_version) -+{ -+ xmlNode *hello_node = NULL; -+ xmlNode *hello = NULL; -+ -+ if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name) -+ || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) { -+ crm_err("Could not create IPC hello message from %s (UUID %s): " -+ "missing information", -+ client_name? client_name : "unknown client", -+ uuid? uuid : "unknown"); -+ return NULL; -+ } -+ -+ hello_node = create_xml_node(NULL, XML_TAG_OPTIONS); -+ if (hello_node == NULL) { -+ crm_err("Could not create IPC hello message from %s (UUID %s): " -+ "Message data creation failed", client_name, uuid); -+ return NULL; -+ } -+ -+ crm_xml_add(hello_node, "major_version", major_version); -+ crm_xml_add(hello_node, "minor_version", minor_version); -+ crm_xml_add(hello_node, "client_name", client_name); -+ crm_xml_add(hello_node, "client_uuid", uuid); -+ -+ hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid); -+ if (hello == NULL) { -+ crm_err("Could not create IPC hello message from %s (UUID %s): " -+ "Request creation failed", client_name, uuid); -+ return NULL; -+ } -+ free_xml(hello_node); -+ -+ crm_trace("Created hello message from %s (UUID %s)", client_name, uuid); -+ return hello; -+} -diff --git a/lib/common/ipc_common.c b/lib/common/ipc_common.c -new file mode 100644 -index 0000000..78360aa ---- /dev/null -+++ b/lib/common/ipc_common.c -@@ -0,0 +1,103 @@ -+/* -+ * Copyright 2004-2020 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. -+ * -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -+ */ -+ -+#include -+ -+#include -+#include -+ -+#include -+#include "crmcommon_private.h" -+ -+#define MIN_MSG_SIZE 12336 // sizeof(struct qb_ipc_connection_response) -+#define MAX_MSG_SIZE 128*1024 // 128k default -+ -+/*! -+ * \internal -+ * \brief Choose an IPC buffer size in bytes -+ * -+ * \param[in] max Use this value if environment/default is lower -+ * -+ * \return Maximum of max and value of PCMK_ipc_buffer (default 128KB) -+ */ -+unsigned int -+pcmk__ipc_buffer_size(unsigned int max) -+{ -+ static unsigned int global_max = 0; -+ -+ if (global_max == 0) { -+ const char *env = getenv("PCMK_ipc_buffer"); -+ -+ if (env) { -+ int env_max = crm_parse_int(env, "0"); -+ -+ global_max = (env_max > 0)? QB_MAX(MIN_MSG_SIZE, env_max) : MAX_MSG_SIZE; -+ -+ } else { -+ global_max = MAX_MSG_SIZE; -+ } -+ } -+ return QB_MAX(max, global_max); -+} -+ -+/*! -+ * \brief Return pacemaker's default IPC buffer size -+ * -+ * \return IPC buffer size in bytes -+ */ -+unsigned int -+crm_ipc_default_buffer_size(void) -+{ -+ static unsigned int default_size = 0; -+ -+ if (default_size == 0) { -+ default_size = pcmk__ipc_buffer_size(0); -+ } -+ return default_size; -+} -+ -+/*! -+ * \internal -+ * \brief Check whether an IPC header is valid -+ * -+ * \param[in] header IPC header to check -+ * -+ * \return true if IPC header has a supported version, false otherwise -+ */ -+bool -+pcmk__valid_ipc_header(const pcmk__ipc_header_t *header) -+{ -+ if (header == NULL) { -+ crm_err("IPC message without header"); -+ return false; -+ -+ } else if (header->version > PCMK__IPC_VERSION) { -+ crm_err("Filtering incompatible v%d IPC message (only versions <= %d supported)", -+ header->version, PCMK__IPC_VERSION); -+ return false; -+ } -+ return true; -+} -+ -+const char * -+pcmk__client_type_str(enum pcmk__client_type client_type) -+{ -+ switch (client_type) { -+ case PCMK__CLIENT_IPC: -+ return "IPC"; -+ case PCMK__CLIENT_TCP: -+ return "TCP"; -+#ifdef HAVE_GNUTLS_GNUTLS_H -+ case PCMK__CLIENT_TLS: -+ return "TLS"; -+#endif -+ default: -+ return "unknown"; -+ } -+} -diff --git a/lib/common/ipc_server.c b/lib/common/ipc_server.c -new file mode 100644 -index 0000000..b747be3 ---- /dev/null -+++ b/lib/common/ipc_server.c -@@ -0,0 +1,903 @@ -+/* -+ * Copyright 2004-2020 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. -+ * -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include "crmcommon_private.h" -+ -+/* Evict clients whose event queue grows this large (by default) */ -+#define PCMK_IPC_DEFAULT_QUEUE_MAX 500 -+ -+static GHashTable *client_connections = NULL; -+ -+/*! -+ * \internal -+ * \brief Count IPC clients -+ * -+ * \return Number of active IPC client connections -+ */ -+guint -+pcmk__ipc_client_count() -+{ -+ return client_connections? g_hash_table_size(client_connections) : 0; -+} -+ -+/*! -+ * \internal -+ * \brief Execute a function for each active IPC client connection -+ * -+ * \param[in] func Function to call -+ * \param[in] user_data Pointer to pass to function -+ * -+ * \note The parameters are the same as for g_hash_table_foreach(). -+ */ -+void -+pcmk__foreach_ipc_client(GHFunc func, gpointer user_data) -+{ -+ if ((func != NULL) && (client_connections != NULL)) { -+ g_hash_table_foreach(client_connections, func, user_data); -+ } -+} -+ -+/*! -+ * \internal -+ * \brief Remote IPC clients based on iterative function result -+ * -+ * \param[in] func Function to call for each active IPC client -+ * \param[in] user_data Pointer to pass to function -+ * -+ * \note The parameters are the same as for g_hash_table_foreach_remove(). -+ */ -+void -+pcmk__foreach_ipc_client_remove(GHRFunc func, gpointer user_data) -+{ -+ if ((func != NULL) && (client_connections != NULL)) { -+ g_hash_table_foreach_remove(client_connections, func, user_data); -+ } -+} -+ -+pcmk__client_t * -+pcmk__find_client(qb_ipcs_connection_t *c) -+{ -+ if (client_connections) { -+ return g_hash_table_lookup(client_connections, c); -+ } -+ -+ crm_trace("No client found for %p", c); -+ return NULL; -+} -+ -+pcmk__client_t * -+pcmk__find_client_by_id(const char *id) -+{ -+ gpointer key; -+ pcmk__client_t *client; -+ GHashTableIter iter; -+ -+ if (client_connections && id) { -+ g_hash_table_iter_init(&iter, client_connections); -+ while (g_hash_table_iter_next(&iter, &key, (gpointer *) & client)) { -+ if (strcmp(client->id, id) == 0) { -+ return client; -+ } -+ } -+ } -+ -+ crm_trace("No client found with id=%s", id); -+ return NULL; -+} -+ -+const char * -+pcmk__client_name(pcmk__client_t *c) -+{ -+ if (c == NULL) { -+ return "null"; -+ } else if (c->name == NULL && c->id == NULL) { -+ return "unknown"; -+ } else if (c->name == NULL) { -+ return c->id; -+ } else { -+ return c->name; -+ } -+} -+ -+void -+pcmk__client_cleanup(void) -+{ -+ if (client_connections != NULL) { -+ int active = g_hash_table_size(client_connections); -+ -+ if (active) { -+ crm_err("Exiting with %d active IPC client%s", -+ active, pcmk__plural_s(active)); -+ } -+ g_hash_table_destroy(client_connections); client_connections = NULL; -+ } -+} -+ -+void -+pcmk__drop_all_clients(qb_ipcs_service_t *service) -+{ -+ qb_ipcs_connection_t *c = NULL; -+ -+ if (service == NULL) { -+ return; -+ } -+ -+ c = qb_ipcs_connection_first_get(service); -+ -+ while (c != NULL) { -+ qb_ipcs_connection_t *last = c; -+ -+ c = qb_ipcs_connection_next_get(service, last); -+ -+ /* There really shouldn't be anyone connected at this point */ -+ crm_notice("Disconnecting client %p, pid=%d...", -+ last, pcmk__client_pid(last)); -+ qb_ipcs_disconnect(last); -+ qb_ipcs_connection_unref(last); -+ } -+} -+ -+/*! -+ * \internal -+ * \brief Allocate a new pcmk__client_t object based on an IPC connection -+ * -+ * \param[in] c IPC connection (or NULL to allocate generic client) -+ * \param[in] key Connection table key (or NULL to use sane default) -+ * \param[in] uid_client UID corresponding to c (ignored if c is NULL) -+ * -+ * \return Pointer to new pcmk__client_t (or NULL on error) -+ */ -+static pcmk__client_t * -+client_from_connection(qb_ipcs_connection_t *c, void *key, uid_t uid_client) -+{ -+ pcmk__client_t *client = calloc(1, sizeof(pcmk__client_t)); -+ -+ if (client == NULL) { -+ crm_perror(LOG_ERR, "Allocating client"); -+ return NULL; -+ } -+ -+ if (c) { -+#if ENABLE_ACL -+ client->user = pcmk__uid2username(uid_client); -+ if (client->user == NULL) { -+ client->user = strdup("#unprivileged"); -+ CRM_CHECK(client->user != NULL, free(client); return NULL); -+ crm_err("Unable to enforce ACLs for user ID %d, assuming unprivileged", -+ uid_client); -+ } -+#endif -+ client->ipcs = c; -+ client->kind = PCMK__CLIENT_IPC; -+ client->pid = pcmk__client_pid(c); -+ if (key == NULL) { -+ key = c; -+ } -+ } -+ -+ client->id = crm_generate_uuid(); -+ if (client->id == NULL) { -+ crm_err("Could not generate UUID for client"); -+ free(client->user); -+ free(client); -+ return NULL; -+ } -+ if (key == NULL) { -+ key = client->id; -+ } -+ if (client_connections == NULL) { -+ crm_trace("Creating IPC client table"); -+ client_connections = g_hash_table_new(g_direct_hash, g_direct_equal); -+ } -+ g_hash_table_insert(client_connections, key, client); -+ return client; -+} -+ -+/*! -+ * \brief Allocate a new pcmk__client_t object and generate its ID -+ * -+ * \param[in] key What to use as connections hash table key (NULL to use ID) -+ * -+ * \return Pointer to new pcmk__client_t (asserts on failure) -+ */ -+pcmk__client_t * -+pcmk__new_unauth_client(void *key) -+{ -+ pcmk__client_t *client = client_from_connection(NULL, key, 0); -+ -+ CRM_ASSERT(client != NULL); -+ return client; -+} -+ -+pcmk__client_t * -+pcmk__new_client(qb_ipcs_connection_t *c, uid_t uid_client, gid_t gid_client) -+{ -+ gid_t uid_cluster = 0; -+ gid_t gid_cluster = 0; -+ -+ pcmk__client_t *client = NULL; -+ -+ CRM_CHECK(c != NULL, return NULL); -+ -+ if (pcmk_daemon_user(&uid_cluster, &gid_cluster) < 0) { -+ static bool need_log = TRUE; -+ -+ if (need_log) { -+ crm_warn("Could not find user and group IDs for user %s", -+ CRM_DAEMON_USER); -+ need_log = FALSE; -+ } -+ } -+ -+ if (uid_client != 0) { -+ crm_trace("Giving group %u access to new IPC connection", gid_cluster); -+ /* Passing -1 to chown(2) means don't change */ -+ qb_ipcs_connection_auth_set(c, -1, gid_cluster, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); -+ } -+ -+ /* TODO: Do our own auth checking, return NULL if unauthorized */ -+ client = client_from_connection(c, NULL, uid_client); -+ if (client == NULL) { -+ return NULL; -+ } -+ -+ if ((uid_client == 0) || (uid_client == uid_cluster)) { -+ /* Remember when a connection came from root or hacluster */ -+ set_bit(client->flags, pcmk__client_privileged); -+ } -+ -+ crm_debug("New IPC client %s for PID %u with uid %d and gid %d", -+ client->id, client->pid, uid_client, gid_client); -+ return client; -+} -+ -+static struct iovec * -+pcmk__new_ipc_event(void) -+{ -+ struct iovec *iov = calloc(2, sizeof(struct iovec)); -+ -+ CRM_ASSERT(iov != NULL); -+ return iov; -+} -+ -+/*! -+ * \brief Free an I/O vector created by pcmk__ipc_prepare_iov() -+ * -+ * \param[in] event I/O vector to free -+ */ -+void -+pcmk_free_ipc_event(struct iovec *event) -+{ -+ if (event != NULL) { -+ free(event[0].iov_base); -+ free(event[1].iov_base); -+ free(event); -+ } -+} -+ -+static void -+free_event(gpointer data) -+{ -+ pcmk_free_ipc_event((struct iovec *) data); -+} -+ -+static void -+add_event(pcmk__client_t *c, struct iovec *iov) -+{ -+ if (c->event_queue == NULL) { -+ c->event_queue = g_queue_new(); -+ } -+ g_queue_push_tail(c->event_queue, iov); -+} -+ -+void -+pcmk__free_client(pcmk__client_t *c) -+{ -+ if (c == NULL) { -+ return; -+ } -+ -+ if (client_connections) { -+ if (c->ipcs) { -+ crm_trace("Destroying %p/%p (%d remaining)", -+ c, c->ipcs, g_hash_table_size(client_connections) - 1); -+ g_hash_table_remove(client_connections, c->ipcs); -+ -+ } else { -+ crm_trace("Destroying remote connection %p (%d remaining)", -+ c, g_hash_table_size(client_connections) - 1); -+ g_hash_table_remove(client_connections, c->id); -+ } -+ } -+ -+ if (c->event_timer) { -+ g_source_remove(c->event_timer); -+ } -+ -+ if (c->event_queue) { -+ crm_debug("Destroying %d events", g_queue_get_length(c->event_queue)); -+ g_queue_free_full(c->event_queue, free_event); -+ } -+ -+ free(c->id); -+ free(c->name); -+ free(c->user); -+ if (c->remote) { -+ if (c->remote->auth_timeout) { -+ g_source_remove(c->remote->auth_timeout); -+ } -+ free(c->remote->buffer); -+ free(c->remote); -+ } -+ free(c); -+} -+ -+/*! -+ * \internal -+ * \brief Raise IPC eviction threshold for a client, if allowed -+ * -+ * \param[in,out] client Client to modify -+ * \param[in] qmax New threshold (as non-NULL string) -+ * -+ * \return TRUE if change was allowed, FALSE otherwise -+ */ -+bool -+pcmk__set_client_queue_max(pcmk__client_t *client, const char *qmax) -+{ -+ if (is_set(client->flags, pcmk__client_privileged)) { -+ long long qmax_int; -+ -+ errno = 0; -+ qmax_int = crm_parse_ll(qmax, NULL); -+ if ((errno == 0) && (qmax_int > 0)) { -+ client->queue_max = (unsigned int) qmax_int; -+ return TRUE; -+ } -+ } -+ return FALSE; -+} -+ -+int -+pcmk__client_pid(qb_ipcs_connection_t *c) -+{ -+ struct qb_ipcs_connection_stats stats; -+ -+ stats.client_pid = 0; -+ qb_ipcs_connection_stats_get(c, &stats, 0); -+ return stats.client_pid; -+} -+ -+/*! -+ * \internal -+ * \brief Retrieve message XML from data read from client IPC -+ * -+ * \param[in] c IPC client connection -+ * \param[in] data Data read from client connection -+ * \param[out] id Where to store message ID from libqb header -+ * \param[out] flags Where to store flags from libqb header -+ * -+ * \return Message XML on success, NULL otherwise -+ */ -+xmlNode * -+pcmk__client_data2xml(pcmk__client_t *c, void *data, uint32_t *id, -+ uint32_t *flags) -+{ -+ xmlNode *xml = NULL; -+ char *uncompressed = NULL; -+ char *text = ((char *)data) + sizeof(pcmk__ipc_header_t); -+ pcmk__ipc_header_t *header = data; -+ -+ if (!pcmk__valid_ipc_header(header)) { -+ return NULL; -+ } -+ -+ if (id) { -+ *id = ((struct qb_ipc_response_header *)data)->id; -+ } -+ if (flags) { -+ *flags = header->flags; -+ } -+ -+ if (is_set(header->flags, crm_ipc_proxied)) { -+ /* Mark this client as being the endpoint of a proxy connection. -+ * Proxy connections responses are sent on the event channel, to avoid -+ * blocking the controller serving as proxy. -+ */ -+ c->flags |= pcmk__client_proxied; -+ } -+ -+ if (header->size_compressed) { -+ int rc = 0; -+ unsigned int size_u = 1 + header->size_uncompressed; -+ uncompressed = calloc(1, size_u); -+ -+ crm_trace("Decompressing message data %u bytes into %u bytes", -+ header->size_compressed, size_u); -+ -+ rc = BZ2_bzBuffToBuffDecompress(uncompressed, &size_u, text, header->size_compressed, 1, 0); -+ text = uncompressed; -+ -+ if (rc != BZ_OK) { -+ crm_err("Decompression failed: %s " CRM_XS " bzerror=%d", -+ bz2_strerror(rc), rc); -+ free(uncompressed); -+ return NULL; -+ } -+ } -+ -+ CRM_ASSERT(text[header->size_uncompressed - 1] == 0); -+ -+ xml = string2xml(text); -+ crm_log_xml_trace(xml, "[IPC received]"); -+ -+ free(uncompressed); -+ return xml; -+} -+ -+static int crm_ipcs_flush_events(pcmk__client_t *c); -+ -+static gboolean -+crm_ipcs_flush_events_cb(gpointer data) -+{ -+ pcmk__client_t *c = data; -+ -+ c->event_timer = 0; -+ crm_ipcs_flush_events(c); -+ return FALSE; -+} -+ -+/*! -+ * \internal -+ * \brief Add progressive delay before next event queue flush -+ * -+ * \param[in,out] c Client connection to add delay to -+ * \param[in] queue_len Current event queue length -+ */ -+static inline void -+delay_next_flush(pcmk__client_t *c, unsigned int queue_len) -+{ -+ /* Delay a maximum of 1.5 seconds */ -+ guint delay = (queue_len < 5)? (1000 + 100 * queue_len) : 1500; -+ -+ c->event_timer = g_timeout_add(delay, crm_ipcs_flush_events_cb, c); -+} -+ -+/*! -+ * \internal -+ * \brief Send client any messages in its queue -+ * -+ * \param[in] c Client to flush -+ * -+ * \return Standard Pacemaker return value -+ */ -+static int -+crm_ipcs_flush_events(pcmk__client_t *c) -+{ -+ int rc = pcmk_rc_ok; -+ ssize_t qb_rc = 0; -+ unsigned int sent = 0; -+ unsigned int queue_len = 0; -+ -+ if (c == NULL) { -+ return rc; -+ -+ } else if (c->event_timer) { -+ /* There is already a timer, wait until it goes off */ -+ crm_trace("Timer active for %p - %d", c->ipcs, c->event_timer); -+ return rc; -+ } -+ -+ if (c->event_queue) { -+ queue_len = g_queue_get_length(c->event_queue); -+ } -+ while (sent < 100) { -+ pcmk__ipc_header_t *header = NULL; -+ struct iovec *event = NULL; -+ -+ if (c->event_queue) { -+ // We don't pop unless send is successful -+ event = g_queue_peek_head(c->event_queue); -+ } -+ if (event == NULL) { // Queue is empty -+ break; -+ } -+ -+ qb_rc = qb_ipcs_event_sendv(c->ipcs, event, 2); -+ if (qb_rc < 0) { -+ rc = (int) -qb_rc; -+ break; -+ } -+ event = g_queue_pop_head(c->event_queue); -+ -+ sent++; -+ header = event[0].iov_base; -+ if (header->size_compressed) { -+ crm_trace("Event %d to %p[%d] (%lld compressed bytes) sent", -+ header->qb.id, c->ipcs, c->pid, (long long) qb_rc); -+ } else { -+ crm_trace("Event %d to %p[%d] (%lld bytes) sent: %.120s", -+ header->qb.id, c->ipcs, c->pid, (long long) qb_rc, -+ (char *) (event[1].iov_base)); -+ } -+ pcmk_free_ipc_event(event); -+ } -+ -+ queue_len -= sent; -+ if (sent > 0 || queue_len) { -+ crm_trace("Sent %d events (%d remaining) for %p[%d]: %s (%lld)", -+ sent, queue_len, c->ipcs, c->pid, -+ pcmk_rc_str(rc), (long long) qb_rc); -+ } -+ -+ if (queue_len) { -+ -+ /* Allow clients to briefly fall behind on processing incoming messages, -+ * but drop completely unresponsive clients so the connection doesn't -+ * consume resources indefinitely. -+ */ -+ if (queue_len > QB_MAX(c->queue_max, PCMK_IPC_DEFAULT_QUEUE_MAX)) { -+ if ((c->queue_backlog <= 1) || (queue_len < c->queue_backlog)) { -+ /* Don't evict for a new or shrinking backlog */ -+ crm_warn("Client with process ID %u has a backlog of %u messages " -+ CRM_XS " %p", c->pid, queue_len, c->ipcs); -+ } else { -+ crm_err("Evicting client with process ID %u due to backlog of %u messages " -+ CRM_XS " %p", c->pid, queue_len, c->ipcs); -+ c->queue_backlog = 0; -+ qb_ipcs_disconnect(c->ipcs); -+ return rc; -+ } -+ } -+ -+ c->queue_backlog = queue_len; -+ delay_next_flush(c, queue_len); -+ -+ } else { -+ /* Event queue is empty, there is no backlog */ -+ c->queue_backlog = 0; -+ } -+ -+ return rc; -+} -+ -+/*! -+ * \internal -+ * \brief Create an I/O vector for sending an IPC XML message -+ * -+ * \param[in] request Identifier for libqb response header -+ * \param[in] message XML message to send -+ * \param[in] max_send_size If 0, default IPC buffer size is used -+ * \param[out] result Where to store prepared I/O vector -+ * \param[out] bytes Size of prepared data in bytes -+ * -+ * \return Standard Pacemaker return code -+ */ -+int -+pcmk__ipc_prepare_iov(uint32_t request, xmlNode *message, -+ uint32_t max_send_size, struct iovec **result, -+ ssize_t *bytes) -+{ -+ static unsigned int biggest = 0; -+ struct iovec *iov; -+ unsigned int total = 0; -+ char *compressed = NULL; -+ char *buffer = NULL; -+ pcmk__ipc_header_t *header = NULL; -+ -+ if ((message == NULL) || (result == NULL)) { -+ return EINVAL; -+ } -+ -+ header = calloc(1, sizeof(pcmk__ipc_header_t)); -+ if (header == NULL) { -+ return ENOMEM; /* errno mightn't be set by allocator */ -+ } -+ -+ buffer = dump_xml_unformatted(message); -+ -+ if (max_send_size == 0) { -+ max_send_size = crm_ipc_default_buffer_size(); -+ } -+ CRM_LOG_ASSERT(max_send_size != 0); -+ -+ *result = NULL; -+ iov = pcmk__new_ipc_event(); -+ iov[0].iov_len = sizeof(pcmk__ipc_header_t); -+ iov[0].iov_base = header; -+ -+ header->version = PCMK__IPC_VERSION; -+ header->size_uncompressed = 1 + strlen(buffer); -+ total = iov[0].iov_len + header->size_uncompressed; -+ -+ if (total < max_send_size) { -+ iov[1].iov_base = buffer; -+ iov[1].iov_len = header->size_uncompressed; -+ -+ } else { -+ unsigned int new_size = 0; -+ -+ if (pcmk__compress(buffer, (unsigned int) header->size_uncompressed, -+ (unsigned int) max_send_size, &compressed, -+ &new_size) == pcmk_rc_ok) { -+ -+ header->flags |= crm_ipc_compressed; -+ header->size_compressed = new_size; -+ -+ iov[1].iov_len = header->size_compressed; -+ iov[1].iov_base = compressed; -+ -+ free(buffer); -+ -+ biggest = QB_MAX(header->size_compressed, biggest); -+ -+ } else { -+ crm_log_xml_trace(message, "EMSGSIZE"); -+ biggest = QB_MAX(header->size_uncompressed, biggest); -+ -+ crm_err("Could not compress %u-byte message into less than IPC " -+ "limit of %u bytes; set PCMK_ipc_buffer to higher value " -+ "(%u bytes suggested)", -+ header->size_uncompressed, max_send_size, 4 * biggest); -+ -+ free(compressed); -+ free(buffer); -+ pcmk_free_ipc_event(iov); -+ return EMSGSIZE; -+ } -+ } -+ -+ header->qb.size = iov[0].iov_len + iov[1].iov_len; -+ header->qb.id = (int32_t)request; /* Replying to a specific request */ -+ -+ *result = iov; -+ CRM_ASSERT(header->qb.size > 0); -+ if (bytes != NULL) { -+ *bytes = header->qb.size; -+ } -+ return pcmk_rc_ok; -+} -+ -+int -+pcmk__ipc_send_iov(pcmk__client_t *c, struct iovec *iov, uint32_t flags) -+{ -+ int rc = pcmk_rc_ok; -+ static uint32_t id = 1; -+ pcmk__ipc_header_t *header = iov[0].iov_base; -+ -+ if (c->flags & pcmk__client_proxied) { -+ /* _ALL_ replies to proxied connections need to be sent as events */ -+ if (is_not_set(flags, crm_ipc_server_event)) { -+ flags |= crm_ipc_server_event; -+ /* this flag lets us know this was originally meant to be a response. -+ * even though we're sending it over the event channel. */ -+ flags |= crm_ipc_proxied_relay_response; -+ } -+ } -+ -+ header->flags |= flags; -+ if (flags & crm_ipc_server_event) { -+ header->qb.id = id++; /* We don't really use it, but doesn't hurt to set one */ -+ -+ if (flags & crm_ipc_server_free) { -+ crm_trace("Sending the original to %p[%d]", c->ipcs, c->pid); -+ add_event(c, iov); -+ -+ } else { -+ struct iovec *iov_copy = pcmk__new_ipc_event(); -+ -+ crm_trace("Sending a copy to %p[%d]", c->ipcs, c->pid); -+ iov_copy[0].iov_len = iov[0].iov_len; -+ iov_copy[0].iov_base = malloc(iov[0].iov_len); -+ memcpy(iov_copy[0].iov_base, iov[0].iov_base, iov[0].iov_len); -+ -+ iov_copy[1].iov_len = iov[1].iov_len; -+ iov_copy[1].iov_base = malloc(iov[1].iov_len); -+ memcpy(iov_copy[1].iov_base, iov[1].iov_base, iov[1].iov_len); -+ -+ add_event(c, iov_copy); -+ } -+ -+ } else { -+ ssize_t qb_rc; -+ -+ CRM_LOG_ASSERT(header->qb.id != 0); /* Replying to a specific request */ -+ -+ qb_rc = qb_ipcs_response_sendv(c->ipcs, iov, 2); -+ if (qb_rc < header->qb.size) { -+ if (qb_rc < 0) { -+ rc = (int) -qb_rc; -+ } -+ crm_notice("Response %d to pid %d failed: %s " -+ CRM_XS " bytes=%u rc=%lld ipcs=%p", -+ header->qb.id, c->pid, pcmk_rc_str(rc), -+ header->qb.size, (long long) qb_rc, c->ipcs); -+ -+ } else { -+ crm_trace("Response %d sent, %lld bytes to %p[%d]", -+ header->qb.id, (long long) qb_rc, c->ipcs, c->pid); -+ } -+ -+ if (flags & crm_ipc_server_free) { -+ pcmk_free_ipc_event(iov); -+ } -+ } -+ -+ if (flags & crm_ipc_server_event) { -+ rc = crm_ipcs_flush_events(c); -+ } else { -+ crm_ipcs_flush_events(c); -+ } -+ -+ if ((rc == EPIPE) || (rc == ENOTCONN)) { -+ crm_trace("Client %p disconnected", c->ipcs); -+ } -+ return rc; -+} -+ -+int -+pcmk__ipc_send_xml(pcmk__client_t *c, uint32_t request, xmlNode *message, -+ uint32_t flags) -+{ -+ struct iovec *iov = NULL; -+ int rc = pcmk_rc_ok; -+ -+ if (c == NULL) { -+ return EINVAL; -+ } -+ rc = pcmk__ipc_prepare_iov(request, message, crm_ipc_default_buffer_size(), -+ &iov, NULL); -+ if (rc == pcmk_rc_ok) { -+ rc = pcmk__ipc_send_iov(c, iov, flags | crm_ipc_server_free); -+ } else { -+ pcmk_free_ipc_event(iov); -+ crm_notice("IPC message to pid %d failed: %s " CRM_XS " rc=%d", -+ c->pid, pcmk_rc_str(rc), rc); -+ } -+ return rc; -+} -+ -+void -+pcmk__ipc_send_ack_as(const char *function, int line, pcmk__client_t *c, -+ uint32_t request, uint32_t flags, const char *tag) -+{ -+ if (flags & crm_ipc_client_response) { -+ xmlNode *ack = create_xml_node(NULL, tag); -+ -+ crm_trace("Ack'ing IPC message from %s", pcmk__client_name(c)); -+ c->request_id = 0; -+ crm_xml_add(ack, "function", function); -+ crm_xml_add_int(ack, "line", line); -+ pcmk__ipc_send_xml(c, request, ack, flags); -+ free_xml(ack); -+ } -+} -+ -+/*! -+ * \internal -+ * \brief Add an IPC server to the main loop for the pacemaker-based API -+ * -+ * \param[out] ipcs_ro New IPC server for read-only pacemaker-based API -+ * \param[out] ipcs_rw New IPC server for read/write pacemaker-based API -+ * \param[out] ipcs_shm New IPC server for shared-memory pacemaker-based API -+ * \param[in] ro_cb IPC callbacks for read-only API -+ * \param[in] rw_cb IPC callbacks for read/write and shared-memory APIs -+ * -+ * \note This function exits fatally if unable to create the servers. -+ */ -+void pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro, -+ qb_ipcs_service_t **ipcs_rw, -+ qb_ipcs_service_t **ipcs_shm, -+ struct qb_ipcs_service_handlers *ro_cb, -+ struct qb_ipcs_service_handlers *rw_cb) -+{ -+ *ipcs_ro = mainloop_add_ipc_server(PCMK__SERVER_BASED_RO, -+ QB_IPC_NATIVE, ro_cb); -+ -+ *ipcs_rw = mainloop_add_ipc_server(PCMK__SERVER_BASED_RW, -+ QB_IPC_NATIVE, rw_cb); -+ -+ *ipcs_shm = mainloop_add_ipc_server(PCMK__SERVER_BASED_SHM, -+ QB_IPC_SHM, rw_cb); -+ -+ if (*ipcs_ro == NULL || *ipcs_rw == NULL || *ipcs_shm == NULL) { -+ crm_err("Failed to create the CIB manager: exiting and inhibiting respawn"); -+ crm_warn("Verify pacemaker and pacemaker_remote are not both enabled"); -+ crm_exit(CRM_EX_FATAL); -+ } -+} -+ -+/*! -+ * \internal -+ * \brief Destroy IPC servers for pacemaker-based API -+ * -+ * \param[out] ipcs_ro IPC server for read-only pacemaker-based API -+ * \param[out] ipcs_rw IPC server for read/write pacemaker-based API -+ * \param[out] ipcs_shm IPC server for shared-memory pacemaker-based API -+ * -+ * \note This is a convenience function for calling qb_ipcs_destroy() for each -+ * argument. -+ */ -+void -+pcmk__stop_based_ipc(qb_ipcs_service_t *ipcs_ro, -+ qb_ipcs_service_t *ipcs_rw, -+ qb_ipcs_service_t *ipcs_shm) -+{ -+ qb_ipcs_destroy(ipcs_ro); -+ qb_ipcs_destroy(ipcs_rw); -+ qb_ipcs_destroy(ipcs_shm); -+} -+ -+/*! -+ * \internal -+ * \brief Add an IPC server to the main loop for the pacemaker-controld API -+ * -+ * \param[in] cb IPC callbacks -+ * -+ * \return Newly created IPC server -+ */ -+qb_ipcs_service_t * -+pcmk__serve_controld_ipc(struct qb_ipcs_service_handlers *cb) -+{ -+ return mainloop_add_ipc_server(CRM_SYSTEM_CRMD, QB_IPC_NATIVE, cb); -+} -+ -+/*! -+ * \internal -+ * \brief Add an IPC server to the main loop for the pacemaker-attrd API -+ * -+ * \param[in] cb IPC callbacks -+ * -+ * \note This function exits fatally if unable to create the servers. -+ */ -+void -+pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs, -+ struct qb_ipcs_service_handlers *cb) -+{ -+ *ipcs = mainloop_add_ipc_server(T_ATTRD, QB_IPC_NATIVE, cb); -+ -+ if (*ipcs == NULL) { -+ crm_err("Failed to create pacemaker-attrd server: exiting and inhibiting respawn"); -+ crm_warn("Verify pacemaker and pacemaker_remote are not both enabled."); -+ crm_exit(CRM_EX_FATAL); -+ } -+} -+ -+/*! -+ * \internal -+ * \brief Add an IPC server to the main loop for the pacemaker-fenced API -+ * -+ * \param[in] cb IPC callbacks -+ * -+ * \note This function exits fatally if unable to create the servers. -+ */ -+void -+pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs, -+ struct qb_ipcs_service_handlers *cb) -+{ -+ *ipcs = mainloop_add_ipc_server_with_prio("stonith-ng", QB_IPC_NATIVE, cb, -+ QB_LOOP_HIGH); -+ -+ if (*ipcs == NULL) { -+ crm_err("Failed to create fencer: exiting and inhibiting respawn."); -+ crm_warn("Verify pacemaker and pacemaker_remote are not both enabled."); -+ crm_exit(CRM_EX_FATAL); -+ } -+} -diff --git a/lib/common/messages.c b/lib/common/messages.c -new file mode 100644 -index 0000000..c5b5739 ---- /dev/null -+++ b/lib/common/messages.c -@@ -0,0 +1,146 @@ -+/* -+ * Copyright 2004-2020 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. -+ * -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -+ */ -+ -+#include -+ -+#include -+#include -+ -+#include -+ -+/*! -+ * \brief Create a Pacemaker request (for IPC or cluster layer) -+ * -+ * \param[in] task What to set as the request's task -+ * \param[in] msg_data What to add as the request's data contents -+ * \param[in] host_to What to set as the request's destination host -+ * \param[in] sys_to What to set as the request's destination system -+ * \param[in] sys_from If not NULL, set as request's origin system -+ * \param[in] uuid_from If not NULL, use in request's origin system -+ * \param[in] origin Name of function that called this one -+ * -+ * \return XML of new request -+ * -+ * \note One of sys_from or uuid_from must be non-NULL -+ * \note This function should not be called directly, but via the -+ * create_request() wrapper. -+ * \note The caller is responsible for freeing the result using free_xml(). -+ */ -+xmlNode * -+create_request_adv(const char *task, xmlNode * msg_data, -+ const char *host_to, const char *sys_to, -+ const char *sys_from, const char *uuid_from, -+ const char *origin) -+{ -+ static uint ref_counter = 0; -+ -+ char *true_from = NULL; -+ xmlNode *request = NULL; -+ char *reference = crm_strdup_printf("%s-%s-%lld-%u", -+ (task? task : "_empty_"), -+ (sys_from? sys_from : "_empty_"), -+ (long long) time(NULL), ref_counter++); -+ -+ if (uuid_from != NULL) { -+ true_from = generate_hash_key(sys_from, uuid_from); -+ } else if (sys_from != NULL) { -+ true_from = strdup(sys_from); -+ } else { -+ crm_err("No sys from specified"); -+ } -+ -+ // host_from will get set for us if necessary by the controller when routed -+ request = create_xml_node(NULL, __FUNCTION__); -+ crm_xml_add(request, F_CRM_ORIGIN, origin); -+ crm_xml_add(request, F_TYPE, T_CRM); -+ crm_xml_add(request, F_CRM_VERSION, CRM_FEATURE_SET); -+ crm_xml_add(request, F_CRM_MSG_TYPE, XML_ATTR_REQUEST); -+ crm_xml_add(request, F_CRM_REFERENCE, reference); -+ crm_xml_add(request, F_CRM_TASK, task); -+ crm_xml_add(request, F_CRM_SYS_TO, sys_to); -+ crm_xml_add(request, F_CRM_SYS_FROM, true_from); -+ -+ /* HOSTTO will be ignored if it is to the DC anyway. */ -+ if (host_to != NULL && strlen(host_to) > 0) { -+ crm_xml_add(request, F_CRM_HOST_TO, host_to); -+ } -+ -+ if (msg_data != NULL) { -+ add_message_xml(request, F_CRM_DATA, msg_data); -+ } -+ free(reference); -+ free(true_from); -+ -+ return request; -+} -+ -+/*! -+ * \brief Create a Pacemaker reply (for IPC or cluster layer) -+ * -+ * \param[in] original_request XML of request this is a reply to -+ * \param[in] xml_response_data XML to copy as data section of reply -+ * \param[in] origin Name of function that called this one -+ * -+ * \return XML of new reply -+ * -+ * \note This function should not be called directly, but via the -+ * create_reply() wrapper. -+ * \note The caller is responsible for freeing the result using free_xml(). -+ */ -+xmlNode * -+create_reply_adv(xmlNode *original_request, xmlNode *xml_response_data, -+ const char *origin) -+{ -+ xmlNode *reply = NULL; -+ -+ const char *host_from = crm_element_value(original_request, F_CRM_HOST_FROM); -+ const char *sys_from = crm_element_value(original_request, F_CRM_SYS_FROM); -+ const char *sys_to = crm_element_value(original_request, F_CRM_SYS_TO); -+ const char *type = crm_element_value(original_request, F_CRM_MSG_TYPE); -+ const char *operation = crm_element_value(original_request, F_CRM_TASK); -+ const char *crm_msg_reference = crm_element_value(original_request, F_CRM_REFERENCE); -+ -+ if (type == NULL) { -+ crm_err("Cannot create new_message, no message type in original message"); -+ CRM_ASSERT(type != NULL); -+ return NULL; -+#if 0 -+ } else if (strcasecmp(XML_ATTR_REQUEST, type) != 0) { -+ crm_err("Cannot create new_message, original message was not a request"); -+ return NULL; -+#endif -+ } -+ reply = create_xml_node(NULL, __FUNCTION__); -+ if (reply == NULL) { -+ crm_err("Cannot create new_message, malloc failed"); -+ return NULL; -+ } -+ -+ crm_xml_add(reply, F_CRM_ORIGIN, origin); -+ crm_xml_add(reply, F_TYPE, T_CRM); -+ crm_xml_add(reply, F_CRM_VERSION, CRM_FEATURE_SET); -+ crm_xml_add(reply, F_CRM_MSG_TYPE, XML_ATTR_RESPONSE); -+ crm_xml_add(reply, F_CRM_REFERENCE, crm_msg_reference); -+ crm_xml_add(reply, F_CRM_TASK, operation); -+ -+ /* since this is a reply, we reverse the from and to */ -+ crm_xml_add(reply, F_CRM_SYS_TO, sys_from); -+ crm_xml_add(reply, F_CRM_SYS_FROM, sys_to); -+ -+ /* HOSTTO will be ignored if it is to the DC anyway. */ -+ if (host_from != NULL && strlen(host_from) > 0) { -+ crm_xml_add(reply, F_CRM_HOST_TO, host_from); -+ } -+ -+ if (xml_response_data != NULL) { -+ add_message_xml(reply, F_CRM_DATA, xml_response_data); -+ } -+ -+ return reply; -+} --- -1.8.3.1 - - -From 8cfa76d840aff62b2376136a7ddb1cb54d070458 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 7 Apr 2020 11:39:31 -0500 -Subject: [PATCH 3/8] Refactor: libcrmcommon: move get_message_xml() and - add_message_xml() definition - -recently added messages.c is more logical place than xml.c ---- - lib/common/messages.c | 20 ++++++++++++++++++++ - lib/common/xml.c | 17 ----------------- - 2 files changed, 20 insertions(+), 17 deletions(-) - -diff --git a/lib/common/messages.c b/lib/common/messages.c -index c5b5739..25569db 100644 ---- a/lib/common/messages.c -+++ b/lib/common/messages.c -@@ -12,6 +12,9 @@ - #include - #include - -+#include -+#include -+ - #include - - /*! -@@ -144,3 +147,20 @@ create_reply_adv(xmlNode *original_request, xmlNode *xml_response_data, - - return reply; - } -+ -+xmlNode * -+get_message_xml(xmlNode *msg, const char *field) -+{ -+ xmlNode *tmp = first_named_child(msg, field); -+ -+ return __xml_first_child(tmp); -+} -+ -+gboolean -+add_message_xml(xmlNode *msg, const char *field, xmlNode *xml) -+{ -+ xmlNode *holder = create_xml_node(msg, field); -+ -+ add_node_copy(holder, xml); -+ return TRUE; -+} -diff --git a/lib/common/xml.c b/lib/common/xml.c -index de0c508..e071f8d 100644 ---- a/lib/common/xml.c -+++ b/lib/common/xml.c -@@ -2613,23 +2613,6 @@ write_xml_file(xmlNode * xml_node, const char *filename, gboolean compress) - return write_xml_stream(xml_node, filename, stream, compress); - } - --xmlNode * --get_message_xml(xmlNode * msg, const char *field) --{ -- xmlNode *tmp = first_named_child(msg, field); -- -- return __xml_first_child(tmp); --} -- --gboolean --add_message_xml(xmlNode * msg, const char *field, xmlNode * xml) --{ -- xmlNode *holder = create_xml_node(msg, field); -- -- add_node_copy(holder, xml); -- return TRUE; --} -- - static char * - crm_xml_escape_shuffle(char *text, int start, int *length, const char *replace) - { --- -1.8.3.1 - - -From a123da0a978a45b08f0723fd651059a13d26235c Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 8 Apr 2020 11:09:00 -0500 -Subject: [PATCH 4/8] Refactor: libcrmcommon: drop generate_hash_key() - -The internal one-liner was only used in one location, and that wasn't even to -generate a hash key. ---- - include/crm_internal.h | 2 -- - lib/common/messages.c | 5 +++-- - lib/common/utils.c | 11 ----------- - 3 files changed, 3 insertions(+), 15 deletions(-) - -diff --git a/include/crm_internal.h b/include/crm_internal.h -index 15f9d2b..fd56fc6 100644 ---- a/include/crm_internal.h -+++ b/include/crm_internal.h -@@ -68,8 +68,6 @@ crm_set_bit(const char *function, int line, const char *target, long long word, - # define set_bit(word, bit) word = crm_set_bit(__FUNCTION__, __LINE__, NULL, word, bit) - # define clear_bit(word, bit) word = crm_clear_bit(__FUNCTION__, __LINE__, NULL, word, bit) - --char *generate_hash_key(const char *crm_msg_reference, const char *sys); -- - void strip_text_nodes(xmlNode * xml); - void pcmk_panic(const char *origin); - pid_t pcmk_locate_sbd(void); -diff --git a/lib/common/messages.c b/lib/common/messages.c -index 25569db..d3fa894 100644 ---- a/lib/common/messages.c -+++ b/lib/common/messages.c -@@ -51,11 +51,12 @@ create_request_adv(const char *task, xmlNode * msg_data, - (long long) time(NULL), ref_counter++); - - if (uuid_from != NULL) { -- true_from = generate_hash_key(sys_from, uuid_from); -+ true_from = crm_strdup_printf("%s_%s", uuid_from, -+ (sys_from? sys_from : "none")); - } else if (sys_from != NULL) { - true_from = strdup(sys_from); - } else { -- crm_err("No sys from specified"); -+ crm_err("Cannot create IPC request: No originating system specified"); - } - - // host_from will get set for us if necessary by the controller when routed -diff --git a/lib/common/utils.c b/lib/common/utils.c -index 13e7cb2..0ac96b8 100644 ---- a/lib/common/utils.c -+++ b/lib/common/utils.c -@@ -116,17 +116,6 @@ score2char(int score) - return crm_itoa(score); - } - --char * --generate_hash_key(const char *crm_msg_reference, const char *sys) --{ -- char *hash_key = crm_strdup_printf("%s_%s", (sys? sys : "none"), -- crm_msg_reference); -- -- crm_trace("created hash key: (%s)", hash_key); -- return hash_key; --} -- -- - int - crm_user_lookup(const char *name, uid_t * uid, gid_t * gid) - { --- -1.8.3.1 - - -From dd81ff4b826c662654d6760f20f3ec74f6ae6020 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 16 Apr 2020 16:47:53 -0500 -Subject: [PATCH 5/8] Low: libcrmcommon: new function for draining and quitting - a main loop - -We have an existing drain function to drain events based on a check function -and a timeout, which is better suited to daemons. This one drains up to N -events and then quits the main loop, which is better suited to tools. ---- - include/crm/common/mainloop.h | 3 ++- - lib/common/mainloop.c | 22 ++++++++++++++++++++++ - 2 files changed, 24 insertions(+), 1 deletion(-) - -diff --git a/include/crm/common/mainloop.h b/include/crm/common/mainloop.h -index b443b4e..9957b25 100644 ---- a/include/crm/common/mainloop.h -+++ b/include/crm/common/mainloop.h -@@ -1,5 +1,5 @@ - /* -- * Copyright 2009-2019 the Pacemaker project contributors -+ * Copyright 2009-2020 the Pacemaker project contributors - * - * The version control history for this file may have further details. - * -@@ -146,6 +146,7 @@ pid_t mainloop_child_pid(mainloop_child_t * child); - void mainloop_clear_child_userdata(mainloop_child_t * child); - gboolean mainloop_child_kill(pid_t pid); - -+void pcmk_quit_main_loop(GMainLoop *mloop, unsigned int n); - void pcmk_drain_main_loop(GMainLoop *mloop, guint timer_ms, - bool (*check)(guint)); - -diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c -index 10450e4..634eead 100644 ---- a/lib/common/mainloop.c -+++ b/lib/common/mainloop.c -@@ -1345,6 +1345,28 @@ drain_timeout_cb(gpointer user_data) - } - - /*! -+ * \brief Drain some remaining main loop events then quit it -+ * -+ * \param[in] mloop Main loop to drain and quit -+ * \param[in] n Drain up to this many pending events -+ */ -+void -+pcmk_quit_main_loop(GMainLoop *mloop, unsigned int n) -+{ -+ if ((mloop != NULL) && g_main_loop_is_running(mloop)) { -+ GMainContext *ctx = g_main_loop_get_context(mloop); -+ -+ /* Drain up to n events in case some memory clean-up is pending -+ * (helpful to reduce noise in valgrind output). -+ */ -+ for (int i = 0; (i < n) && g_main_context_pending(ctx); ++i) { -+ g_main_context_dispatch(ctx); -+ } -+ g_main_loop_quit(mloop); -+ } -+} -+ -+/*! - * \brief Process main loop events while a certain condition is met - * - * \param[in] mloop Main loop to process --- -1.8.3.1 - - -From 771a0c56df363eb91fa7eb6ac4b3ee833c5dcd5e Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 14 Apr 2020 13:04:45 -0500 -Subject: [PATCH 6/8] Low: tools: handle memory cleanup better in crm_node - -crm_node had paths that would exit without returning to the main(), -missing the memory cleanup at the end. ---- - tools/crm_node.c | 30 ++++++++++++++---------------- - 1 file changed, 14 insertions(+), 16 deletions(-) - -diff --git a/tools/crm_node.c b/tools/crm_node.c -index 34511f3..db31f20 100644 ---- a/tools/crm_node.c -+++ b/tools/crm_node.c -@@ -187,15 +187,6 @@ new_mainloop_for_ipc(const char *system, ipc_dispatch_fn dispatch) - return ipc; - } - --static void --run_mainloop_and_exit(void) --{ -- g_main_loop_run(mainloop); -- g_main_loop_unref(mainloop); -- mainloop = NULL; -- crm_node_exit(exit_code); --} -- - static int - send_controller_hello(crm_ipc_t *controller) - { -@@ -328,7 +319,9 @@ run_controller_mainloop(uint32_t nodeid) - } - - // Run main loop to get controller reply via dispatch_controller() -- run_mainloop_and_exit(); -+ g_main_loop_run(mainloop); -+ g_main_loop_unref(mainloop); -+ mainloop = NULL; - } - - static void -@@ -339,7 +332,8 @@ print_node_name(void) - - if (name != NULL) { - printf("%s\n", name); -- crm_node_exit(CRM_EX_OK); -+ exit_code = CRM_EX_OK; -+ return; - - } else { - // Otherwise ask the controller -@@ -486,11 +480,11 @@ remove_node(const char *target_uname) - if (tools_remove_node_cache(node_name, nodeid, daemons[d])) { - crm_err("Failed to connect to %s to remove node '%s'", - daemons[d], target_uname); -- crm_node_exit(CRM_EX_ERROR); -+ exit_code = CRM_EX_ERROR; - return; - } - } -- crm_node_exit(CRM_EX_OK); -+ exit_code = CRM_EX_OK; - } - - static gint -@@ -513,7 +507,8 @@ node_mcp_dispatch(const char *buffer, ssize_t length, gpointer userdata) - - if (msg == NULL) { - fprintf(stderr, "error: Could not understand pacemakerd response\n"); -- crm_node_exit(CRM_EX_PROTOCOL); -+ exit_code = CRM_EX_PROTOCOL; -+ g_main_loop_quit(mainloop); - return 0; - } - -@@ -544,7 +539,8 @@ node_mcp_dispatch(const char *buffer, ssize_t length, gpointer userdata) - } - - free_xml(msg); -- crm_node_exit(CRM_EX_OK); -+ exit_code = CRM_EX_OK; -+ g_main_loop_quit(mainloop); - return 0; - } - -@@ -562,7 +558,9 @@ run_pacemakerd_mainloop(void) - free_xml(poke); - - // Handle reply via node_mcp_dispatch() -- run_mainloop_and_exit(); -+ g_main_loop_run(mainloop); -+ g_main_loop_unref(mainloop); -+ mainloop = NULL; - } - - static GOptionContext * --- -1.8.3.1 - - -From ba7abdbe6cd7f2fb73ce096c87f0599e27000bee Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 20 Apr 2020 15:30:40 -0500 -Subject: [PATCH 7/8] Refactor: tools: use proper type for glib timeout value - in crmadmin - ---- - tools/crmadmin.c | 10 ++++++---- - 1 file changed, 6 insertions(+), 4 deletions(-) - -diff --git a/tools/crmadmin.c b/tools/crmadmin.c -index c58de59..bd4bfe5 100644 ---- a/tools/crmadmin.c -+++ b/tools/crmadmin.c -@@ -19,6 +19,7 @@ - #include - #include - #include -+#include // gboolean, GMainLoop, etc. - - #include - #include -@@ -28,9 +29,10 @@ - - #include - --static int message_timer_id = -1; --static int message_timeout_ms = 30 * 1000; -+#define DEFAULT_MESSAGE_TIMEOUT_MS 30000 - -+static guint message_timer_id = 0; -+static guint message_timeout_ms = DEFAULT_MESSAGE_TIMEOUT_MS; - static GMainLoop *mainloop = NULL; - static crm_ipc_t *crmd_channel = NULL; - static char *admin_uuid = NULL; -@@ -172,9 +174,9 @@ main(int argc, char **argv) - crm_bump_log_level(argc, argv); - break; - case 't': -- message_timeout_ms = atoi(optarg); -+ message_timeout_ms = (guint) atoi(optarg); - if (message_timeout_ms < 1) { -- message_timeout_ms = 30 * 1000; -+ message_timeout_ms = DEFAULT_MESSAGE_TIMEOUT_MS; - } - break; - --- -1.8.3.1 - - -From 27d763920a1e12b9b5747c1b64a5dc1395c58768 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 20 Apr 2020 15:44:20 -0500 -Subject: [PATCH 8/8] Refactor: tools: functionize listing nodes from CIB in - crmadmin - ---- - tools/crmadmin.c | 47 +++++++++++++++++++++++++++-------------------- - 1 file changed, 27 insertions(+), 20 deletions(-) - -diff --git a/tools/crmadmin.c b/tools/crmadmin.c -index bd4bfe5..3e9e959 100644 ---- a/tools/crmadmin.c -+++ b/tools/crmadmin.c -@@ -149,6 +149,32 @@ static pcmk__cli_option_t long_options[] = { - { 0, 0, 0, 0 } - }; - -+// \return Standard Pacemaker return code -+static int -+list_nodes() -+{ -+ cib_t *the_cib = cib_new(); -+ xmlNode *output = NULL; -+ int rc; -+ -+ if (the_cib == NULL) { -+ return ENOMEM; -+ } -+ rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command); -+ if (rc != pcmk_ok) { -+ return pcmk_legacy2rc(rc); -+ } -+ -+ rc = the_cib->cmds->query(the_cib, NULL, &output, -+ cib_scope_local | cib_sync_call); -+ if (rc == pcmk_ok) { -+ do_find_node_list(output); -+ free_xml(output); -+ } -+ the_cib->cmds->signoff(the_cib); -+ return pcmk_legacy2rc(rc); -+} -+ - int - main(int argc, char **argv) - { -@@ -304,26 +330,7 @@ do_work(void) - crmd_operation = CRM_OP_PING; - - } else if (DO_NODE_LIST) { -- -- cib_t *the_cib = cib_new(); -- xmlNode *output = NULL; -- -- int rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command); -- -- if (rc != pcmk_ok) { -- fprintf(stderr, "Could not connect to CIB: %s\n", -- pcmk_strerror(rc)); -- return -1; -- } -- -- rc = the_cib->cmds->query(the_cib, NULL, &output, cib_scope_local | cib_sync_call); -- if(rc == pcmk_ok) { -- do_find_node_list(output); -- -- free_xml(output); -- } -- the_cib->cmds->signoff(the_cib); -- crm_exit(crm_errno2exit(rc)); -+ crm_exit(pcmk_rc2exitc(list_nodes())); - - } else if (DO_RESET) { - /* tell dest_node to initiate the shutdown procedure --- -1.8.3.1 - diff --git a/SOURCES/007-ipc_model.patch b/SOURCES/007-ipc_model.patch deleted file mode 100644 index 55fed07..0000000 --- a/SOURCES/007-ipc_model.patch +++ /dev/null @@ -1,4146 +0,0 @@ -From e34421b2608f235c6a77eb6de4596d93a2128be1 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 3 Apr 2020 11:13:52 -0500 -Subject: [PATCH 1/6] Refactor: libcrmcommon: new model for daemon IPC API - -This is based on tools/crm_resource_controller.[ch] and the node removal code -in tools/crm_node.c, but generalized for any daemon. As of this commit, no -specific daemon is supported. ---- - include/crm/common/internal.h | 8 + - include/crm/common/ipc.h | 95 +++++- - lib/common/crmcommon_private.h | 86 +++++ - lib/common/ipc_client.c | 703 ++++++++++++++++++++++++++++++++++++++++- - lib/common/mainloop.c | 70 ++-- - 5 files changed, 940 insertions(+), 22 deletions(-) - -diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h -index 28b20b4..13c29ea 100644 ---- a/include/crm/common/internal.h -+++ b/include/crm/common/internal.h -@@ -19,6 +19,7 @@ - #include // xmlNode - - #include // crm_strdup_printf() -+#include // mainloop_io_t, struct ipc_client_callbacks - - // Internal ACL-related utilities (from acl.c) - -@@ -103,6 +104,13 @@ pcmk__open_devnull(int flags) - } while (0) - - -+/* internal main loop utilities (from mainloop.c) */ -+ -+int pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata, -+ struct ipc_client_callbacks *callbacks, -+ mainloop_io_t **source); -+ -+ - /* internal procfs utilities (from procfs.c) */ - - pid_t pcmk__procfs_pid_of(const char *name); -diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h -index a0df956..8dee1b1 100644 ---- a/include/crm/common/ipc.h -+++ b/include/crm/common/ipc.h -@@ -16,7 +16,8 @@ extern "C" { - - /** - * \file -- * \brief Wrappers for and extensions to libqb IPC -+ * \brief IPC interface to Pacemaker daemons -+ * - * \ingroup core - */ - -@@ -48,6 +49,96 @@ xmlNode *create_request_adv(const char *task, xmlNode *xml_data, - const char *origin); - - -+/* -+ * The library supports two methods of creating IPC connections. The older code -+ * allows connecting to any arbitrary IPC name. The newer code only allows -+ * connecting to one of the Pacemaker daemons. -+ * -+ * As daemons are converted to use the new model, the old functions should be -+ * considered deprecated for use with those daemons. Once all daemons are -+ * converted, the old functions should be officially deprecated as public API -+ * and eventually made internal API. -+ */ -+ -+/* -+ * Pacemaker daemon IPC -+ */ -+ -+//! Available IPC interfaces -+enum pcmk_ipc_server { -+ pcmk_ipc_attrd, //!< Attribute manager -+ pcmk_ipc_based, //!< CIB manager -+ pcmk_ipc_controld, //!< Controller -+ pcmk_ipc_execd, //!< Executor -+ pcmk_ipc_fenced, //!< Fencer -+ pcmk_ipc_pacemakerd, //!< Launcher -+ pcmk_ipc_schedulerd, //!< Scheduler -+}; -+ -+//! Possible event types that an IPC event callback can be called for -+enum pcmk_ipc_event { -+ pcmk_ipc_event_connect, //!< Result of asynchronous connection attempt -+ pcmk_ipc_event_disconnect, //!< Termination of IPC connection -+ pcmk_ipc_event_reply, //!< Daemon's reply to client IPC request -+ pcmk_ipc_event_notify, //!< Notification from daemon -+}; -+ -+//! How IPC replies should be dispatched -+enum pcmk_ipc_dispatch { -+ pcmk_ipc_dispatch_main, //!< Attach IPC to GMainLoop for dispatch -+ pcmk_ipc_dispatch_poll, //!< Caller will poll and dispatch IPC -+ pcmk_ipc_dispatch_sync, //!< Sending a command will wait for any reply -+}; -+ -+//! Client connection to Pacemaker IPC -+typedef struct pcmk_ipc_api_s pcmk_ipc_api_t; -+ -+/*! -+ * \brief Callback function type for Pacemaker daemon IPC APIs -+ * -+ * \param[in] api IPC API connection -+ * \param[in] event_type The type of event that occurred -+ * \param[in] status Event status -+ * \param[in] event_data Event-specific data -+ * \param[in] user_data Caller data provided when callback was registered -+ * -+ * \note For connection and disconnection events, event_data may be NULL (for -+ * local IPC) or the name of the connected node (for remote IPC, for -+ * daemons that support that). For reply and notify events, event_data is -+ * defined by the specific daemon API. -+ */ -+typedef void (*pcmk_ipc_callback_t)(pcmk_ipc_api_t *api, -+ enum pcmk_ipc_event event_type, -+ crm_exit_t status, -+ void *event_data, void *user_data); -+ -+int pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server); -+ -+void pcmk_free_ipc_api(pcmk_ipc_api_t *api); -+ -+int pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type); -+ -+void pcmk_disconnect_ipc(pcmk_ipc_api_t *api); -+ -+int pcmk_poll_ipc(pcmk_ipc_api_t *api, int timeout_ms); -+ -+void pcmk_dispatch_ipc(pcmk_ipc_api_t *api); -+ -+void pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb, -+ void *user_data); -+ -+const char *pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log); -+ -+bool pcmk_ipc_is_connected(pcmk_ipc_api_t *api); -+ -+int pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name, -+ uint32_t nodeid); -+ -+ -+/* -+ * Generic IPC API (to eventually be deprecated as public API and made internal) -+ */ -+ - /* *INDENT-OFF* */ - enum crm_ipc_flags - { -@@ -58,7 +149,7 @@ enum crm_ipc_flags - crm_ipc_proxied = 0x00000100, /* _ALL_ replies to proxied connections need to be sent as events */ - crm_ipc_client_response = 0x00000200, /* A Response is expected in reply */ - -- // These are options only for pcmk__ipc_send_iov() -+ // These are options for Pacemaker's internal use only (pcmk__ipc_send_*()) - crm_ipc_server_event = 0x00010000, /* Send an Event instead of a Response */ - crm_ipc_server_free = 0x00020000, /* Free the iovec after sending */ - crm_ipc_proxied_relay_response = 0x00040000, /* all replies to proxied connections are sent as events, this flag preserves whether the event should be treated as an actual event, or a response.*/ -diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h -index d06fa20..f9df27d 100644 ---- a/lib/common/crmcommon_private.h -+++ b/lib/common/crmcommon_private.h -@@ -103,6 +103,84 @@ pcmk__xml_attr_value(const xmlAttr *attr) - - #define PCMK__IPC_VERSION 1 - -+// IPC behavior that varies by daemon -+typedef struct pcmk__ipc_methods_s { -+ /*! -+ * \internal -+ * \brief Allocate any private data needed by daemon IPC -+ * -+ * \param[in] api IPC API connection -+ * -+ * \return Standard Pacemaker return code -+ */ -+ int (*new_data)(pcmk_ipc_api_t *api); -+ -+ /*! -+ * \internal -+ * \brief Free any private data used by daemon IPC -+ * -+ * \param[in] api_data Data allocated by new_data() method -+ */ -+ void (*free_data)(void *api_data); -+ -+ /*! -+ * \internal -+ * \brief Perform daemon-specific handling after successful connection -+ * -+ * Some daemons require clients to register before sending any other -+ * commands. The controller requires a CRM_OP_HELLO (with no reply), and -+ * the CIB manager, executor, and fencer require a CRM_OP_REGISTER (with a -+ * reply). Ideally this would be consistent across all daemons, but for now -+ * this allows each to do its own authorization. -+ * -+ * \param[in] api IPC API connection -+ * -+ * \return Standard Pacemaker return code -+ */ -+ int (*post_connect)(pcmk_ipc_api_t *api); -+ -+ /*! -+ * \internal -+ * \brief Check whether an IPC request results in a reply -+ * -+ * \parma[in] api IPC API connection -+ * \param[in] request IPC request XML -+ * -+ * \return true if request would result in an IPC reply, false otherwise -+ */ -+ bool (*reply_expected)(pcmk_ipc_api_t *api, xmlNode *request); -+ -+ /*! -+ * \internal -+ * \brief Perform daemon-specific handling of an IPC message -+ * -+ * \param[in] api IPC API connection -+ * \param[in] msg Message read from IPC connection -+ */ -+ void (*dispatch)(pcmk_ipc_api_t *api, xmlNode *msg); -+ -+ /*! -+ * \internal -+ * \brief Perform daemon-specific handling of an IPC disconnect -+ * -+ * \param[in] api IPC API connection -+ */ -+ void (*post_disconnect)(pcmk_ipc_api_t *api); -+} pcmk__ipc_methods_t; -+ -+// Implementation of pcmk_ipc_api_t -+struct pcmk_ipc_api_s { -+ enum pcmk_ipc_server server; // Daemon this IPC API instance is for -+ enum pcmk_ipc_dispatch dispatch_type; // How replies should be dispatched -+ crm_ipc_t *ipc; // IPC connection -+ mainloop_io_t *mainloop_io; // If using mainloop, I/O source for IPC -+ bool free_on_disconnect; // Whether disconnect should free object -+ pcmk_ipc_callback_t cb; // Caller-registered callback (if any) -+ void *user_data; // Caller-registered data (if any) -+ void *api_data; // For daemon-specific use -+ pcmk__ipc_methods_t *cmds; // Behavior that varies by daemon -+}; -+ - typedef struct pcmk__ipc_header_s { - struct qb_ipc_response_header qb; - uint32_t size_uncompressed; -@@ -112,6 +190,14 @@ typedef struct pcmk__ipc_header_s { - } pcmk__ipc_header_t; - - G_GNUC_INTERNAL -+int pcmk__send_ipc_request(pcmk_ipc_api_t *api, xmlNode *request); -+ -+G_GNUC_INTERNAL -+void pcmk__call_ipc_callback(pcmk_ipc_api_t *api, -+ enum pcmk_ipc_event event_type, -+ crm_exit_t status, void *event_data); -+ -+G_GNUC_INTERNAL - unsigned int pcmk__ipc_buffer_size(unsigned int max); - - G_GNUC_INTERNAL -diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c -index 7737588..16dc9b5 100644 ---- a/lib/common/ipc_client.c -+++ b/lib/common/ipc_client.c -@@ -31,6 +31,679 @@ - #include - #include "crmcommon_private.h" - -+/*! -+ * \brief Create a new object for using Pacemaker daemon IPC -+ * -+ * \param[out] api Where to store new IPC object -+ * \param[in] server Which Pacemaker daemon the object is for -+ * -+ * \return Standard Pacemaker result code -+ * -+ * \note The caller is responsible for freeing *api using pcmk_free_ipc_api(). -+ * \note This is intended to supersede crm_ipc_new() but is not yet usable. -+ */ -+int -+pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server) -+{ -+ size_t max_size = 0; -+ -+ if (api == NULL) { -+ return EINVAL; -+ } -+ -+ *api = calloc(1, sizeof(pcmk_ipc_api_t)); -+ if (*api == NULL) { -+ return errno; -+ } -+ -+ (*api)->server = server; -+ if (pcmk_ipc_name(*api, false) == NULL) { -+ pcmk_free_ipc_api(*api); -+ *api = NULL; -+ return EOPNOTSUPP; -+ } -+ -+ // Set server methods and max_size (if not default) -+ switch (server) { -+ case pcmk_ipc_attrd: -+ break; -+ -+ case pcmk_ipc_based: -+ max_size = 512 * 1024; // 512KB -+ break; -+ -+ case pcmk_ipc_controld: -+ break; -+ -+ case pcmk_ipc_execd: -+ break; -+ -+ case pcmk_ipc_fenced: -+ break; -+ -+ case pcmk_ipc_pacemakerd: -+ break; -+ -+ case pcmk_ipc_schedulerd: -+ // @TODO max_size could vary by client, maybe take as argument? -+ max_size = 5 * 1024 * 1024; // 5MB -+ break; -+ } -+ if ((*api)->cmds == NULL) { -+ pcmk_free_ipc_api(*api); -+ *api = NULL; -+ return ENOMEM; -+ } -+ -+ (*api)->ipc = crm_ipc_new(pcmk_ipc_name(*api, false), max_size); -+ if ((*api)->ipc == NULL) { -+ pcmk_free_ipc_api(*api); -+ *api = NULL; -+ return ENOMEM; -+ } -+ -+ // If daemon API has its own data to track, allocate it -+ if ((*api)->cmds->new_data != NULL) { -+ if ((*api)->cmds->new_data(*api) != pcmk_rc_ok) { -+ pcmk_free_ipc_api(*api); -+ *api = NULL; -+ return ENOMEM; -+ } -+ } -+ crm_trace("Created %s API IPC object", pcmk_ipc_name(*api, true)); -+ return pcmk_rc_ok; -+} -+ -+static void -+free_daemon_specific_data(pcmk_ipc_api_t *api) -+{ -+ if ((api != NULL) && (api->cmds != NULL)) { -+ if ((api->cmds->free_data != NULL) && (api->api_data != NULL)) { -+ api->cmds->free_data(api->api_data); -+ api->api_data = NULL; -+ } -+ free(api->cmds); -+ api->cmds = NULL; -+ } -+} -+ -+/*! -+ * \internal -+ * \brief Call an IPC API event callback, if one is registed -+ * -+ * \param[in] api IPC API connection -+ * \param[in] event_type The type of event that occurred -+ * \param[in] status Event status -+ * \param[in] event_data Event-specific data -+ */ -+void -+pcmk__call_ipc_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type, -+ crm_exit_t status, void *event_data) -+{ -+ if ((api != NULL) && (api->cb != NULL)) { -+ api->cb(api, event_type, status, event_data, api->user_data); -+ } -+} -+ -+/*! -+ * \internal -+ * \brief Clean up after an IPC disconnect -+ * -+ * \param[in] user_data IPC API connection that disconnected -+ * -+ * \note This function can be used as a main loop IPC destroy callback. -+ */ -+static void -+ipc_post_disconnect(gpointer user_data) -+{ -+ pcmk_ipc_api_t *api = user_data; -+ -+ crm_info("Disconnected from %s IPC API", pcmk_ipc_name(api, true)); -+ -+ // Perform any daemon-specific handling needed -+ if ((api->cmds != NULL) && (api->cmds->post_disconnect != NULL)) { -+ api->cmds->post_disconnect(api); -+ } -+ -+ // Call client's registered event callback -+ pcmk__call_ipc_callback(api, pcmk_ipc_event_disconnect, CRM_EX_DISCONNECT, -+ NULL); -+ -+ /* If this is being called from a running main loop, mainloop_gio_destroy() -+ * will free ipc and mainloop_io immediately after calling this function. -+ * If this is called from a stopped main loop, these will leak, so the best -+ * practice is to close the connection before stopping the main loop. -+ */ -+ api->ipc = NULL; -+ api->mainloop_io = NULL; -+ -+ if (api->free_on_disconnect) { -+ /* pcmk_free_ipc_api() has already been called, but did not free api -+ * or api->cmds because this function needed them. Do that now. -+ */ -+ free_daemon_specific_data(api); -+ crm_trace("Freeing IPC API object after disconnect"); -+ free(api); -+ } -+} -+ -+/*! -+ * \brief Free the contents of an IPC API object -+ * -+ * \param[in] api IPC API object to free -+ */ -+void -+pcmk_free_ipc_api(pcmk_ipc_api_t *api) -+{ -+ bool free_on_disconnect = false; -+ -+ if (api == NULL) { -+ return; -+ } -+ crm_debug("Releasing %s IPC API", pcmk_ipc_name(api, true)); -+ -+ if (api->ipc != NULL) { -+ if (api->mainloop_io != NULL) { -+ /* We need to keep the api pointer itself around, because it is the -+ * user data for the IPC client destroy callback. That will be -+ * triggered by the pcmk_disconnect_ipc() call below, but it might -+ * happen later in the main loop (if still running). -+ * -+ * This flag tells the destroy callback to free the object. It can't -+ * do that unconditionally, because the application might call this -+ * function after a disconnect that happened by other means. -+ */ -+ free_on_disconnect = api->free_on_disconnect = true; -+ } -+ pcmk_disconnect_ipc(api); // Frees api if free_on_disconnect is true -+ } -+ if (!free_on_disconnect) { -+ free_daemon_specific_data(api); -+ crm_trace("Freeing IPC API object"); -+ free(api); -+ } -+} -+ -+/*! -+ * \brief Get the IPC name used with an IPC API connection -+ * -+ * \param[in] api IPC API connection -+ * \param[in] for_log If true, return human-friendly name instead of IPC name -+ * -+ * \return IPC API's human-friendly or connection name, or if none is available, -+ * "Pacemaker" if for_log is true and NULL if for_log is false -+ */ -+const char * -+pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log) -+{ -+ if (api == NULL) { -+ return for_log? "Pacemaker" : NULL; -+ } -+ switch (api->server) { -+ case pcmk_ipc_attrd: -+ return for_log? "attribute manager" : NULL /* T_ATTRD */; -+ -+ case pcmk_ipc_based: -+ return for_log? "CIB manager" : NULL /* PCMK__SERVER_BASED_RW */; -+ -+ case pcmk_ipc_controld: -+ return for_log? "controller" : NULL /* CRM_SYSTEM_CRMD */; -+ -+ case pcmk_ipc_execd: -+ return for_log? "executor" : NULL /* CRM_SYSTEM_LRMD */; -+ -+ case pcmk_ipc_fenced: -+ return for_log? "fencer" : NULL /* "stonith-ng" */; -+ -+ case pcmk_ipc_pacemakerd: -+ return for_log? "launcher" : NULL /* CRM_SYSTEM_MCP */; -+ -+ case pcmk_ipc_schedulerd: -+ return for_log? "scheduler" : NULL /* CRM_SYSTEM_PENGINE */; -+ -+ default: -+ return for_log? "Pacemaker" : NULL; -+ } -+} -+ -+/*! -+ * \brief Check whether an IPC API connection is active -+ * -+ * \param[in] api IPC API connection -+ * -+ * \return true if IPC is connected, false otherwise -+ */ -+bool -+pcmk_ipc_is_connected(pcmk_ipc_api_t *api) -+{ -+ return (api != NULL) && crm_ipc_connected(api->ipc); -+} -+ -+/*! -+ * \internal -+ * \brief Call the daemon-specific API's dispatch function -+ * -+ * Perform daemon-specific handling of IPC reply dispatch. It is the daemon -+ * method's responsibility to call the client's registered event callback, as -+ * well as allocate and free any event data. -+ * -+ * \param[in] api IPC API connection -+ */ -+static void -+call_api_dispatch(pcmk_ipc_api_t *api, xmlNode *message) -+{ -+ crm_log_xml_trace(message, "ipc-received"); -+ if ((api->cmds != NULL) && (api->cmds->dispatch != NULL)) { -+ api->cmds->dispatch(api, message); -+ } -+} -+ -+/*! -+ * \internal -+ * \brief Dispatch data read from IPC source -+ * -+ * \param[in] buffer Data read from IPC -+ * \param[in] length Number of bytes of data in buffer (ignored) -+ * \param[in] user_data IPC object -+ * -+ * \return Always 0 (meaning connection is still required) -+ * -+ * \note This function can be used as a main loop IPC dispatch callback. -+ */ -+static int -+dispatch_ipc_data(const char *buffer, ssize_t length, gpointer user_data) -+{ -+ pcmk_ipc_api_t *api = user_data; -+ xmlNode *msg; -+ -+ CRM_CHECK(api != NULL, return 0); -+ -+ if (buffer == NULL) { -+ crm_warn("Empty message received from %s IPC", -+ pcmk_ipc_name(api, true)); -+ return 0; -+ } -+ -+ msg = string2xml(buffer); -+ if (msg == NULL) { -+ crm_warn("Malformed message received from %s IPC", -+ pcmk_ipc_name(api, true)); -+ return 0; -+ } -+ call_api_dispatch(api, msg); -+ free_xml(msg); -+ return 0; -+} -+ -+/*! -+ * \brief Check whether an IPC connection has data available (without main loop) -+ * -+ * \param[in] api IPC API connection -+ * \param[in] timeout_ms If less than 0, poll indefinitely; if 0, poll once -+ * and return immediately; otherwise, poll for up to -+ * this many milliseconds -+ * -+ * \return Standard Pacemaker return code -+ * -+ * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call -+ * this function to check whether IPC data is available. Return values of -+ * interest include pcmk_rc_ok meaning data is available, and EAGAIN -+ * meaning no data is available; all other values indicate errors. -+ * \todo This does not allow the caller to poll multiple file descriptors at -+ * once. If there is demand for that, we could add a wrapper for -+ * crm_ipc_get_fd(api->ipc), so the caller can call poll() themselves. -+ */ -+int -+pcmk_poll_ipc(pcmk_ipc_api_t *api, int timeout_ms) -+{ -+ int rc; -+ struct pollfd pollfd = { 0, }; -+ -+ if ((api == NULL) || (api->dispatch_type != pcmk_ipc_dispatch_poll)) { -+ return EINVAL; -+ } -+ pollfd.fd = crm_ipc_get_fd(api->ipc); -+ pollfd.events = POLLIN; -+ rc = poll(&pollfd, 1, timeout_ms); -+ if (rc < 0) { -+ return errno; -+ } else if (rc == 0) { -+ return EAGAIN; -+ } -+ return pcmk_rc_ok; -+} -+ -+/*! -+ * \brief Dispatch available messages on an IPC connection (without main loop) -+ * -+ * \param[in] api IPC API connection -+ * -+ * \return Standard Pacemaker return code -+ * -+ * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call -+ * this function when IPC data is available. -+ */ -+void -+pcmk_dispatch_ipc(pcmk_ipc_api_t *api) -+{ -+ if (api == NULL) { -+ return; -+ } -+ while (crm_ipc_ready(api->ipc)) { -+ if (crm_ipc_read(api->ipc) > 0) { -+ dispatch_ipc_data(crm_ipc_buffer(api->ipc), 0, api); -+ } -+ } -+} -+ -+// \return Standard Pacemaker return code -+static int -+connect_with_main_loop(pcmk_ipc_api_t *api) -+{ -+ int rc; -+ -+ struct ipc_client_callbacks callbacks = { -+ .dispatch = dispatch_ipc_data, -+ .destroy = ipc_post_disconnect, -+ }; -+ -+ rc = pcmk__add_mainloop_ipc(api->ipc, G_PRIORITY_DEFAULT, api, -+ &callbacks, &(api->mainloop_io)); -+ if (rc != pcmk_rc_ok) { -+ return rc; -+ } -+ crm_debug("Connected to %s IPC (attached to main loop)", -+ pcmk_ipc_name(api, true)); -+ /* After this point, api->mainloop_io owns api->ipc, so api->ipc -+ * should not be explicitly freed. -+ */ -+ return pcmk_rc_ok; -+} -+ -+// \return Standard Pacemaker return code -+static int -+connect_without_main_loop(pcmk_ipc_api_t *api) -+{ -+ int rc; -+ -+ if (!crm_ipc_connect(api->ipc)) { -+ rc = errno; -+ crm_ipc_close(api->ipc); -+ return rc; -+ } -+ crm_debug("Connected to %s IPC (without main loop)", -+ pcmk_ipc_name(api, true)); -+ return pcmk_rc_ok; -+} -+ -+/*! -+ * \brief Connect to a Pacemaker daemon via IPC -+ * -+ * \param[in] api IPC API instance -+ * \param[out] dispatch_type How IPC replies should be dispatched -+ * -+ * \return Standard Pacemaker return code -+ */ -+int -+pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type) -+{ -+ int rc = pcmk_rc_ok; -+ -+ if ((api == NULL) || (api->ipc == NULL)) { -+ crm_err("Cannot connect to uninitialized API object"); -+ return EINVAL; -+ } -+ -+ if (crm_ipc_connected(api->ipc)) { -+ crm_trace("Already connected to %s IPC API", pcmk_ipc_name(api, true)); -+ return pcmk_rc_ok; -+ } -+ -+ api->dispatch_type = dispatch_type; -+ switch (dispatch_type) { -+ case pcmk_ipc_dispatch_main: -+ rc = connect_with_main_loop(api); -+ break; -+ -+ case pcmk_ipc_dispatch_sync: -+ case pcmk_ipc_dispatch_poll: -+ rc = connect_without_main_loop(api); -+ break; -+ } -+ if (rc != pcmk_rc_ok) { -+ return rc; -+ } -+ -+ if ((api->cmds != NULL) && (api->cmds->post_connect != NULL)) { -+ rc = api->cmds->post_connect(api); -+ if (rc != pcmk_rc_ok) { -+ crm_ipc_close(api->ipc); -+ } -+ } -+ return rc; -+} -+ -+/*! -+ * \brief Disconnect an IPC API instance -+ * -+ * \param[in] api IPC API connection -+ * -+ * \return Standard Pacemaker return code -+ * -+ * \note If the connection is attached to a main loop, this function should be -+ * called before quitting the main loop, to ensure that all memory is -+ * freed. -+ */ -+void -+pcmk_disconnect_ipc(pcmk_ipc_api_t *api) -+{ -+ if ((api == NULL) || (api->ipc == NULL)) { -+ return; -+ } -+ switch (api->dispatch_type) { -+ case pcmk_ipc_dispatch_main: -+ { -+ mainloop_io_t *mainloop_io = api->mainloop_io; -+ -+ // Make sure no code with access to api can use these again -+ api->mainloop_io = NULL; -+ api->ipc = NULL; -+ -+ mainloop_del_ipc_client(mainloop_io); -+ // After this point api might have already been freed -+ } -+ break; -+ -+ case pcmk_ipc_dispatch_poll: -+ case pcmk_ipc_dispatch_sync: -+ { -+ crm_ipc_t *ipc = api->ipc; -+ -+ // Make sure no code with access to api can use ipc again -+ api->ipc = NULL; -+ -+ // This should always be the case already, but to be safe -+ api->free_on_disconnect = false; -+ -+ crm_ipc_destroy(ipc); -+ ipc_post_disconnect(api); -+ } -+ break; -+ } -+} -+ -+/*! -+ * \brief Register a callback for IPC API events -+ * -+ * \param[in] api IPC API connection -+ * \param[in] callback Callback to register -+ * \param[in] userdata Caller data to pass to callback -+ * -+ * \note This function may be called multiple times to update the callback -+ * and/or user data. The caller remains responsible for freeing -+ * userdata in any case (after the IPC is disconnected, if the -+ * user data is still registered with the IPC). -+ */ -+void -+pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb, -+ void *user_data) -+{ -+ if (api == NULL) { -+ return; -+ } -+ api->cb = cb; -+ api->user_data = user_data; -+} -+ -+/*! -+ * \internal -+ * \brief Send an XML request across an IPC API connection -+ * -+ * \param[in] api IPC API connection -+ * \param[in] request XML request to send -+ * -+ * \return Standard Pacemaker return code -+ * -+ * \note Daemon-specific IPC API functions should call this function to send -+ * requests, because it handles different dispatch types appropriately. -+ */ -+int -+pcmk__send_ipc_request(pcmk_ipc_api_t *api, xmlNode *request) -+{ -+ int rc; -+ xmlNode *reply = NULL; -+ enum crm_ipc_flags flags = crm_ipc_flags_none; -+ -+ if ((api == NULL) || (api->ipc == NULL) || (request == NULL)) { -+ return EINVAL; -+ } -+ crm_log_xml_trace(request, "ipc-sent"); -+ -+ // Synchronous dispatch requires waiting for a reply -+ if ((api->dispatch_type == pcmk_ipc_dispatch_sync) -+ && (api->cmds != NULL) -+ && (api->cmds->reply_expected != NULL) -+ && (api->cmds->reply_expected(api, request))) { -+ flags = crm_ipc_client_response; -+ } -+ -+ // The 0 here means a default timeout of 5 seconds -+ rc = crm_ipc_send(api->ipc, request, flags, 0, &reply); -+ -+ if (rc < 0) { -+ return pcmk_legacy2rc(rc); -+ } else if (rc == 0) { -+ return ENODATA; -+ } -+ -+ // With synchronous dispatch, we dispatch any reply now -+ if (reply != NULL) { -+ call_api_dispatch(api, reply); -+ free_xml(reply); -+ } -+ return pcmk_rc_ok; -+} -+ -+/*! -+ * \internal -+ * \brief Create the XML for an IPC request to purge a node from the peer cache -+ * -+ * \param[in] api IPC API connection -+ * \param[in] node_name If not NULL, name of node to purge -+ * \param[in] nodeid If not 0, node ID of node to purge -+ * -+ * \return Newly allocated IPC request XML -+ * -+ * \note The controller, fencer, and pacemakerd use the same request syntax, but -+ * the attribute manager uses a different one. The CIB manager doesn't -+ * have any syntax for it. The executor and scheduler don't connect to the -+ * cluster layer and thus don't have or need any syntax for it. -+ * -+ * \todo Modify the attribute manager to accept the common syntax (as well -+ * as its current one, for compatibility with older clients). Modify -+ * the CIB manager to accept and honor the common syntax. Modify the -+ * executor and scheduler to accept the syntax (immediately returning -+ * success), just for consistency. Modify this function to use the -+ * common syntax with all daemons if their version supports it. -+ */ -+static xmlNode * -+create_purge_node_request(pcmk_ipc_api_t *api, const char *node_name, -+ uint32_t nodeid) -+{ -+ xmlNode *request = NULL; -+ const char *client = crm_system_name? crm_system_name : "client"; -+ -+ switch (api->server) { -+ case pcmk_ipc_attrd: -+ request = create_xml_node(NULL, __FUNCTION__); -+ crm_xml_add(request, F_TYPE, T_ATTRD); -+ crm_xml_add(request, F_ORIG, crm_system_name); -+ crm_xml_add(request, PCMK__XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE); -+ crm_xml_add(request, PCMK__XA_ATTR_NODE_NAME, node_name); -+ if (nodeid > 0) { -+ crm_xml_add_int(request, PCMK__XA_ATTR_NODE_ID, (int) nodeid); -+ } -+ break; -+ -+ case pcmk_ipc_controld: -+ case pcmk_ipc_fenced: -+ case pcmk_ipc_pacemakerd: -+ request = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, -+ pcmk_ipc_name(api, false), client, NULL); -+ if (nodeid > 0) { -+ crm_xml_set_id(request, "%lu", (unsigned long) nodeid); -+ } -+ crm_xml_add(request, XML_ATTR_UNAME, node_name); -+ break; -+ -+ case pcmk_ipc_based: -+ case pcmk_ipc_execd: -+ case pcmk_ipc_schedulerd: -+ break; -+ } -+ return request; -+} -+ -+/*! -+ * \brief Ask a Pacemaker daemon to purge a node from its peer cache -+ * -+ * \param[in] api IPC API connection -+ * \param[in] node_name If not NULL, name of node to purge -+ * \param[in] nodeid If not 0, node ID of node to purge -+ * -+ * \return Standard Pacemaker return code -+ * -+ * \note At least one of node_name or nodeid must be specified. -+ */ -+int -+pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name, uint32_t nodeid) -+{ -+ int rc = 0; -+ xmlNode *request = NULL; -+ -+ if (api == NULL) { -+ return EINVAL; -+ } -+ if ((node_name == NULL) && (nodeid == 0)) { -+ return EINVAL; -+ } -+ -+ request = create_purge_node_request(api, node_name, nodeid); -+ if (request == NULL) { -+ return EOPNOTSUPP; -+ } -+ rc = pcmk__send_ipc_request(api, request); -+ free_xml(request); -+ -+ crm_debug("%s peer cache purge of node %s[%lu]: rc=%d", -+ pcmk_ipc_name(api, true), node_name, (unsigned long) nodeid, rc); -+ return rc; -+} -+ -+/* -+ * Generic IPC API (to eventually be deprecated as public API and made internal) -+ */ -+ - struct crm_ipc_s { - struct pollfd pfd; - unsigned int max_buf_size; // maximum bytes we can send or receive over IPC -@@ -42,16 +715,44 @@ struct crm_ipc_s { - qb_ipcc_connection_t *ipc; - }; - -+/*! -+ * \brief Create a new (legacy) object for using Pacemaker daemon IPC -+ * -+ * \param[in] name IPC system name to connect to -+ * \param[in] max_size Use a maximum IPC buffer size of at least this size -+ * -+ * \return Newly allocated IPC object on success, NULL otherwise -+ * -+ * \note The caller is responsible for freeing the result using -+ * crm_ipc_destroy(). -+ * \note This should be considered deprecated for use with daemons supported by -+ * pcmk_new_ipc_api(). -+ */ - crm_ipc_t * - crm_ipc_new(const char *name, size_t max_size) - { - crm_ipc_t *client = NULL; - - client = calloc(1, sizeof(crm_ipc_t)); -+ if (client == NULL) { -+ crm_err("Could not create IPC connection: %s", strerror(errno)); -+ return NULL; -+ } - - client->name = strdup(name); -+ if (client->name == NULL) { -+ crm_err("Could not create IPC connection: %s", strerror(errno)); -+ free(client); -+ return NULL; -+ } - client->buf_size = pcmk__ipc_buffer_size(max_size); - client->buffer = malloc(client->buf_size); -+ if (client->buffer == NULL) { -+ crm_err("Could not create IPC connection: %s", strerror(errno)); -+ free(client->name); -+ free(client); -+ return NULL; -+ } - - /* Clients initiating connection pick the max buf size */ - client->max_buf_size = client->buf_size; -@@ -143,8 +844,6 @@ void - crm_ipc_close(crm_ipc_t * client) - { - if (client) { -- crm_trace("Disconnecting %s IPC connection %p (%p)", client->name, client, client->ipc); -- - if (client->ipc) { - qb_ipcc_connection_t *ipc = client->ipc; - -diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c -index 634eead..e942e57 100644 ---- a/lib/common/mainloop.c -+++ b/lib/common/mainloop.c -@@ -834,32 +834,66 @@ mainloop_gio_destroy(gpointer c) - free(c_name); - } - --mainloop_io_t * --mainloop_add_ipc_client(const char *name, int priority, size_t max_size, void *userdata, -- struct ipc_client_callbacks *callbacks) -+/*! -+ * \brief Connect to IPC and add it as a main loop source -+ * -+ * \param[in] ipc IPC connection to add -+ * \param[in] priority Event source priority to use for connection -+ * \param[in] userdata Data to register with callbacks -+ * \param[in] callbacks Dispatch and destroy callbacks for connection -+ * \param[out] source Newly allocated event source -+ * -+ * \return Standard Pacemaker return code -+ * -+ * \note On failure, the caller is still responsible for ipc. On success, the -+ * caller should call mainloop_del_ipc_client() when source is no longer -+ * needed, which will lead to the disconnection of the IPC later in the -+ * main loop if it is connected. However the IPC disconnects, -+ * mainloop_gio_destroy() will free ipc and source after calling the -+ * destroy callback. -+ */ -+int -+pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata, -+ struct ipc_client_callbacks *callbacks, -+ mainloop_io_t **source) - { -- mainloop_io_t *client = NULL; -- crm_ipc_t *conn = crm_ipc_new(name, max_size); -+ CRM_CHECK((ipc != NULL) && (callbacks != NULL), return EINVAL); - -- if (conn && crm_ipc_connect(conn)) { -- int32_t fd = crm_ipc_get_fd(conn); -+ if (!crm_ipc_connect(ipc)) { -+ return ENOTCONN; -+ } -+ *source = mainloop_add_fd(crm_ipc_name(ipc), priority, crm_ipc_get_fd(ipc), -+ userdata, NULL); -+ if (*source == NULL) { -+ int rc = errno; - -- client = mainloop_add_fd(name, priority, fd, userdata, NULL); -+ crm_ipc_close(ipc); -+ return rc; - } -+ (*source)->ipc = ipc; -+ (*source)->destroy_fn = callbacks->destroy; -+ (*source)->dispatch_fn_ipc = callbacks->dispatch; -+ return pcmk_rc_ok; -+} - -- if (client == NULL) { -- crm_perror(LOG_TRACE, "Connection to %s failed", name); -- if (conn) { -- crm_ipc_close(conn); -- crm_ipc_destroy(conn); -+mainloop_io_t * -+mainloop_add_ipc_client(const char *name, int priority, size_t max_size, -+ void *userdata, struct ipc_client_callbacks *callbacks) -+{ -+ crm_ipc_t *ipc = crm_ipc_new(name, max_size); -+ mainloop_io_t *source = NULL; -+ int rc = pcmk__add_mainloop_ipc(ipc, priority, userdata, callbacks, -+ &source); -+ -+ if (rc != pcmk_rc_ok) { -+ if (crm_log_level == LOG_STDOUT) { -+ fprintf(stderr, "Connection to %s failed: %s", -+ name, pcmk_rc_str(rc)); - } -+ crm_ipc_destroy(ipc); - return NULL; - } -- -- client->ipc = conn; -- client->destroy_fn = callbacks->destroy; -- client->dispatch_fn_ipc = callbacks->dispatch; -- return client; -+ return source; - } - - void --- -1.8.3.1 - - -From b9539da27998ff5e6c8b681f39603550a923ca33 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 6 Apr 2020 17:40:55 -0500 -Subject: [PATCH 2/6] Refactor: libcrmcommon: add C API for controller IPC - -Implement a C API for controller IPC using the new IPC API model. ---- - include/crm/common/Makefile.am | 2 +- - include/crm/common/ipc.h | 4 +- - include/crm/common/ipc_controld.h | 99 +++++++ - lib/common/Makefile.am | 1 + - lib/common/crmcommon_private.h | 6 + - lib/common/ipc_client.c | 46 +-- - lib/common/ipc_controld.c | 609 ++++++++++++++++++++++++++++++++++++++ - 7 files changed, 723 insertions(+), 44 deletions(-) - create mode 100644 include/crm/common/ipc_controld.h - create mode 100644 lib/common/ipc_controld.c - -diff --git a/include/crm/common/Makefile.am b/include/crm/common/Makefile.am -index 776e4a7..f29d105 100644 ---- a/include/crm/common/Makefile.am -+++ b/include/crm/common/Makefile.am -@@ -12,7 +12,7 @@ MAINTAINERCLEANFILES = Makefile.in - headerdir=$(pkgincludedir)/crm/common - - header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h results.h \ -- nvpair.h acl.h -+ nvpair.h acl.h ipc_controld.h - noinst_HEADERS = internal.h alerts_internal.h \ - iso8601_internal.h remote_internal.h xml_internal.h \ - ipc_internal.h output.h cmdline_internal.h curses_internal.h \ -diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h -index 8dee1b1..c67aaea 100644 ---- a/include/crm/common/ipc.h -+++ b/include/crm/common/ipc.h -@@ -217,7 +217,9 @@ unsigned int crm_ipc_default_buffer_size(void); - int crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid, - pid_t *gotpid, uid_t *gotuid, gid_t *gotgid); - --/* Utils */ -+/* This is controller-specific but is declared in this header for C API -+ * backward compatibility. -+ */ - xmlNode *create_hello_message(const char *uuid, const char *client_name, - const char *major_version, const char *minor_version); - -diff --git a/include/crm/common/ipc_controld.h b/include/crm/common/ipc_controld.h -new file mode 100644 -index 0000000..0ebabfc ---- /dev/null -+++ b/include/crm/common/ipc_controld.h -@@ -0,0 +1,99 @@ -+/* -+ * Copyright 2020 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. -+ * -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -+ */ -+ -+#ifndef PCMK__IPC_CONTROLD__H -+# define PCMK__IPC_CONTROLD__H -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+/** -+ * \file -+ * \brief IPC commands for Pacemaker controller -+ * -+ * \ingroup core -+ */ -+ -+#include // bool -+#include // xmlNode -+#include // pcmk_ipc_api_t -+ -+//! Possible types of controller replies -+enum pcmk_controld_api_reply { -+ pcmk_controld_reply_unknown, -+ pcmk_controld_reply_reprobe, -+ pcmk_controld_reply_info, -+ pcmk_controld_reply_resource, -+ pcmk_controld_reply_ping, -+}; -+ -+/*! -+ * Controller reply passed to event callback -+ * -+ * \note Shutdown and election calls have no reply. Reprobe calls are -+ * acknowledged but contain no data (reply_type will be the only item -+ * set). Node info and ping calls have their own reply data. Fail and -+ * refresh calls use the resource reply type and reply data. -+ * \note The pointers in the reply are only guaranteed to be meaningful for the -+ * execution of the callback; if the values are needed for later, the -+ * callback should copy them. -+ */ -+typedef struct { -+ enum pcmk_controld_api_reply reply_type; -+ const char *feature_set; //!< CRM feature set advertised by controller -+ const char *host_from; //!< Name of node that sent reply -+ -+ union { -+ // pcmk_controld_reply_info -+ struct { -+ bool have_quorum; -+ bool is_remote; -+ int id; -+ const char *uuid; -+ const char *uname; -+ const char *state; -+ } node_info; -+ -+ // pcmk_controld_reply_resource -+ struct { -+ xmlNode *node_state; //cmds = pcmk__controld_api_methods(); - break; - - case pcmk_ipc_execd: -@@ -247,7 +249,7 @@ pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log) - return for_log? "CIB manager" : NULL /* PCMK__SERVER_BASED_RW */; - - case pcmk_ipc_controld: -- return for_log? "controller" : NULL /* CRM_SYSTEM_CRMD */; -+ return for_log? "controller" : CRM_SYSTEM_CRMD; - - case pcmk_ipc_execd: - return for_log? "executor" : NULL /* CRM_SYSTEM_LRMD */; -@@ -1412,43 +1414,3 @@ bail: - } - return rc; - } -- --xmlNode * --create_hello_message(const char *uuid, -- const char *client_name, const char *major_version, const char *minor_version) --{ -- xmlNode *hello_node = NULL; -- xmlNode *hello = NULL; -- -- if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name) -- || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) { -- crm_err("Could not create IPC hello message from %s (UUID %s): " -- "missing information", -- client_name? client_name : "unknown client", -- uuid? uuid : "unknown"); -- return NULL; -- } -- -- hello_node = create_xml_node(NULL, XML_TAG_OPTIONS); -- if (hello_node == NULL) { -- crm_err("Could not create IPC hello message from %s (UUID %s): " -- "Message data creation failed", client_name, uuid); -- return NULL; -- } -- -- crm_xml_add(hello_node, "major_version", major_version); -- crm_xml_add(hello_node, "minor_version", minor_version); -- crm_xml_add(hello_node, "client_name", client_name); -- crm_xml_add(hello_node, "client_uuid", uuid); -- -- hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid); -- if (hello == NULL) { -- crm_err("Could not create IPC hello message from %s (UUID %s): " -- "Request creation failed", client_name, uuid); -- return NULL; -- } -- free_xml(hello_node); -- -- crm_trace("Created hello message from %s (UUID %s)", client_name, uuid); -- return hello; --} -diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c -new file mode 100644 -index 0000000..22bb733 ---- /dev/null -+++ b/lib/common/ipc_controld.c -@@ -0,0 +1,609 @@ -+/* -+ * Copyright 2020 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. -+ * -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include "crmcommon_private.h" -+ -+struct controld_api_private_s { -+ char *client_uuid; -+ unsigned int replies_expected; -+}; -+ -+// \return Standard Pacemaker return code -+static int -+new_data(pcmk_ipc_api_t *api) -+{ -+ struct controld_api_private_s *private = NULL; -+ -+ api->api_data = calloc(1, sizeof(struct controld_api_private_s)); -+ -+ if (api->api_data == NULL) { -+ return errno; -+ } -+ -+ private = api->api_data; -+ -+ /* This is set to the PID because that's how it was always done, but PIDs -+ * are not unique because clients can be remote. The value appears to be -+ * unused other than as part of F_CRM_SYS_FROM in IPC requests, which is -+ * only compared against the internal system names (CRM_SYSTEM_TENGINE, -+ * etc.), so it shouldn't be a problem. -+ */ -+ private->client_uuid = pcmk__getpid_s(); -+ -+ /* @TODO Implement a call ID model similar to the CIB, executor, and fencer -+ * IPC APIs, so that requests and replies can be matched, and -+ * duplicate replies can be discarded. -+ */ -+ return pcmk_rc_ok; -+} -+ -+static void -+free_data(void *data) -+{ -+ free(((struct controld_api_private_s *) data)->client_uuid); -+ free(data); -+} -+ -+// \return Standard Pacemaker return code -+static int -+post_connect(pcmk_ipc_api_t *api) -+{ -+ /* The controller currently requires clients to register via a hello -+ * request, but does not reply back. -+ */ -+ struct controld_api_private_s *private = api->api_data; -+ const char *client_name = crm_system_name? crm_system_name : "client"; -+ xmlNode *hello; -+ int rc; -+ -+ hello = create_hello_message(private->client_uuid, client_name, -+ PCMK__CONTROLD_API_MAJOR, -+ PCMK__CONTROLD_API_MINOR); -+ rc = pcmk__send_ipc_request(api, hello); -+ free_xml(hello); -+ if (rc != pcmk_rc_ok) { -+ crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s", -+ pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc); -+ } else { -+ crm_debug("Sent IPC hello to %s", pcmk_ipc_name(api, true)); -+ } -+ return rc; -+} -+ -+#define xml_true(xml, field) crm_is_true(crm_element_value(xml, field)) -+ -+static void -+set_node_info_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data) -+{ -+ data->reply_type = pcmk_controld_reply_info; -+ if (msg_data == NULL) { -+ return; -+ } -+ data->data.node_info.have_quorum = xml_true(msg_data, XML_ATTR_HAVE_QUORUM); -+ data->data.node_info.is_remote = xml_true(msg_data, XML_NODE_IS_REMOTE); -+ crm_element_value_int(msg_data, XML_ATTR_ID, &(data->data.node_info.id)); -+ data->data.node_info.uuid = crm_element_value(msg_data, XML_ATTR_UUID); -+ data->data.node_info.uname = crm_element_value(msg_data, XML_ATTR_UNAME); -+ data->data.node_info.state = crm_element_value(msg_data, XML_NODE_IS_PEER); -+} -+ -+static void -+set_ping_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data) -+{ -+ data->reply_type = pcmk_controld_reply_ping; -+ if (msg_data == NULL) { -+ return; -+ } -+ data->data.ping.sys_from = crm_element_value(msg_data, -+ XML_PING_ATTR_SYSFROM); -+ data->data.ping.fsa_state = crm_element_value(msg_data, -+ XML_PING_ATTR_CRMDSTATE); -+ data->data.ping.result = crm_element_value(msg_data, XML_PING_ATTR_STATUS); -+} -+ -+static bool -+reply_expected(pcmk_ipc_api_t *api, xmlNode *request) -+{ -+ const char *command = crm_element_value(request, F_CRM_TASK); -+ -+ if (command == NULL) { -+ return false; -+ } -+ -+ // We only need to handle commands that functions in this file can send -+ return !strcmp(command, CRM_OP_REPROBE) -+ || !strcmp(command, CRM_OP_NODE_INFO) -+ || !strcmp(command, CRM_OP_PING) -+ || !strcmp(command, CRM_OP_LRM_FAIL) -+ || !strcmp(command, CRM_OP_LRM_DELETE); -+} -+ -+static void -+dispatch(pcmk_ipc_api_t *api, xmlNode *reply) -+{ -+ struct controld_api_private_s *private = api->api_data; -+ crm_exit_t status = CRM_EX_OK; -+ xmlNode *msg_data = NULL; -+ const char *value = NULL; -+ pcmk_controld_api_reply_t reply_data = { -+ pcmk_controld_reply_unknown, NULL, NULL, -+ }; -+ -+ if (private->replies_expected > 0) { -+ private->replies_expected--; -+ } -+ -+ // Do some basic validation of the reply -+ -+ /* @TODO We should be able to verify that value is always a response, but -+ * currently the controller doesn't always properly set the type. Even -+ * if we fix the controller, we'll still need to handle replies from -+ * old versions (feature set could be used to differentiate). -+ */ -+ value = crm_element_value(reply, F_CRM_MSG_TYPE); -+ if ((value == NULL) || (strcmp(value, XML_ATTR_REQUEST) -+ && strcmp(value, XML_ATTR_RESPONSE))) { -+ crm_debug("Unrecognizable controller message: invalid message type '%s'", -+ crm_str(value)); -+ status = CRM_EX_PROTOCOL; -+ reply = NULL; -+ } -+ -+ if (crm_element_value(reply, XML_ATTR_REFERENCE) == NULL) { -+ crm_debug("Unrecognizable controller message: no reference"); -+ status = CRM_EX_PROTOCOL; -+ reply = NULL; -+ } -+ -+ value = crm_element_value(reply, F_CRM_TASK); -+ if (value == NULL) { -+ crm_debug("Unrecognizable controller message: no command name"); -+ status = CRM_EX_PROTOCOL; -+ reply = NULL; -+ } -+ -+ // Parse useful info from reply -+ -+ if (reply != NULL) { -+ reply_data.feature_set = crm_element_value(reply, XML_ATTR_VERSION); -+ reply_data.host_from = crm_element_value(reply, F_CRM_HOST_FROM); -+ msg_data = get_message_xml(reply, F_CRM_DATA); -+ -+ if (!strcmp(value, CRM_OP_REPROBE)) { -+ reply_data.reply_type = pcmk_controld_reply_reprobe; -+ -+ } else if (!strcmp(value, CRM_OP_NODE_INFO)) { -+ set_node_info_data(&reply_data, msg_data); -+ -+ } else if (!strcmp(value, CRM_OP_INVOKE_LRM)) { -+ reply_data.reply_type = pcmk_controld_reply_resource; -+ reply_data.data.resource.node_state = msg_data; -+ -+ } else if (!strcmp(value, CRM_OP_PING)) { -+ set_ping_data(&reply_data, msg_data); -+ -+ } else { -+ crm_debug("Unrecognizable controller message: unknown command '%s'", -+ value); -+ status = CRM_EX_PROTOCOL; -+ reply = NULL; -+ } -+ } -+ -+ pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data); -+} -+ -+pcmk__ipc_methods_t * -+pcmk__controld_api_methods() -+{ -+ pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t)); -+ -+ if (cmds != NULL) { -+ cmds->new_data = new_data; -+ cmds->free_data = free_data; -+ cmds->post_connect = post_connect; -+ cmds->reply_expected = reply_expected; -+ cmds->dispatch = dispatch; -+ } -+ return cmds; -+} -+ -+/*! -+ * \internal -+ * \brief Create XML for a controller IPC request -+ * -+ * \param[in] api Controller connection -+ * \param[in] op Controller IPC command name -+ * \param[in] node Node name to set as destination host -+ * \param[in] msg_data XML to attach to request as message data -+ * -+ * \return Newly allocated XML for request -+ */ -+static xmlNode * -+create_controller_request(pcmk_ipc_api_t *api, const char *op, -+ const char *node, xmlNode *msg_data) -+{ -+ struct controld_api_private_s *private = api->api_data; -+ const char *sys_to = NULL; -+ -+ if ((node == NULL) && !strcmp(op, CRM_OP_PING)) { -+ sys_to = CRM_SYSTEM_DC; -+ } else { -+ sys_to = CRM_SYSTEM_CRMD; -+ } -+ return create_request(op, msg_data, node, sys_to, -+ (crm_system_name? crm_system_name : "client"), -+ private->client_uuid); -+} -+ -+// \return Standard Pacemaker return code -+static int -+send_controller_request(pcmk_ipc_api_t *api, xmlNode *request, -+ bool reply_is_expected) -+{ -+ int rc; -+ -+ if (crm_element_value(request, XML_ATTR_REFERENCE) == NULL) { -+ return EINVAL; -+ } -+ rc = pcmk__send_ipc_request(api, request); -+ if ((rc == pcmk_rc_ok) && reply_is_expected) { -+ struct controld_api_private_s *private = api->api_data; -+ -+ private->replies_expected++; -+ } -+ return rc; -+} -+ -+static xmlNode * -+create_reprobe_message_data(const char *target_node, const char *router_node) -+{ -+ xmlNode *msg_data; -+ -+ msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE); -+ crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node); -+ if ((router_node != NULL) && safe_str_neq(router_node, target_node)) { -+ crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node); -+ } -+ return msg_data; -+} -+ -+/*! -+ * \brief Send a reprobe controller operation -+ * -+ * \param[in] api Controller connection -+ * \param[in] target_node Name of node to reprobe -+ * \param[in] router_node Router node for host -+ * -+ * \return Standard Pacemaker return code -+ * \note Event callback will get a reply of type pcmk_controld_reply_reprobe. -+ */ -+int -+pcmk_controld_api_reprobe(pcmk_ipc_api_t *api, const char *target_node, -+ const char *router_node) -+{ -+ xmlNode *request; -+ xmlNode *msg_data; -+ int rc = pcmk_rc_ok; -+ -+ if (api == NULL) { -+ return EINVAL; -+ } -+ if (router_node == NULL) { -+ router_node = target_node; -+ } -+ crm_debug("Sending %s IPC request to reprobe %s via %s", -+ pcmk_ipc_name(api, true), crm_str(target_node), -+ crm_str(router_node)); -+ msg_data = create_reprobe_message_data(target_node, router_node); -+ request = create_controller_request(api, CRM_OP_REPROBE, router_node, -+ msg_data); -+ rc = send_controller_request(api, request, true); -+ free_xml(msg_data); -+ free_xml(request); -+ return rc; -+} -+ -+/*! -+ * \brief Send a "node info" controller operation -+ * -+ * \param[in] api Controller connection -+ * \param[in] nodeid ID of node to get info for (or 0 for local node) -+ * -+ * \return Standard Pacemaker return code -+ * \note Event callback will get a reply of type pcmk_controld_reply_info. -+ */ -+int -+pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid) -+{ -+ xmlNode *request; -+ int rc = pcmk_rc_ok; -+ -+ request = create_controller_request(api, CRM_OP_NODE_INFO, NULL, NULL); -+ if (request == NULL) { -+ return EINVAL; -+ } -+ if (nodeid > 0) { -+ crm_xml_set_id(request, "%lu", (unsigned long) nodeid); -+ } -+ -+ rc = send_controller_request(api, request, true); -+ free_xml(request); -+ return rc; -+} -+ -+/*! -+ * \brief Ask the controller for status -+ * -+ * \param[in] api Controller connection -+ * \param[in] node_name Name of node whose status is desired (or NULL for DC) -+ * -+ * \return Standard Pacemaker return code -+ * \note Event callback will get a reply of type pcmk_controld_reply_ping. -+ */ -+int -+pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name) -+{ -+ xmlNode *request; -+ int rc = pcmk_rc_ok; -+ -+ request = create_controller_request(api, CRM_OP_PING, node_name, NULL); -+ if (request == NULL) { -+ return EINVAL; -+ } -+ rc = send_controller_request(api, request, true); -+ free_xml(request); -+ return rc; -+} -+ -+/*! -+ * \internal -+ * \brief Ask the controller to shut down -+ * -+ * \param[in] api Controller connection -+ * \param[in] node_name Name of node whose controller should shut down -+ * -+ * \return Standard Pacemaker return code -+ * -+ * \note This capability currently does not work, so the function is considered -+ * internal. It will likely be removed. -+ * \note Event callback will not get a reply. -+ */ -+int -+pcmk_controld_api_shutdown(pcmk_ipc_api_t *api, const char *node_name) -+{ -+ xmlNode *request; -+ int rc = pcmk_rc_ok; -+ -+ request = create_controller_request(api, CRM_OP_SHUTDOWN, NULL, NULL); -+ if (request == NULL) { -+ return EINVAL; -+ } -+ rc = send_controller_request(api, request, false); -+ free_xml(request); -+ return rc; -+} -+ -+/*! -+ * \internal -+ * \brief Ask the controller to start a DC election -+ * -+ * \param[in] api Controller connection -+ * -+ * \return Standard Pacemaker return code -+ * -+ * \note This capability currently does not work, so the function is considered -+ * internal. It will likely be removed. -+ * \note Event callback will not get a reply. -+ */ -+int -+pcmk_controld_api_start_election(pcmk_ipc_api_t *api) -+{ -+ xmlNode *request; -+ int rc = pcmk_rc_ok; -+ -+ request = create_controller_request(api, CRM_OP_VOTE, NULL, NULL); -+ if (request == NULL) { -+ return EINVAL; -+ } -+ rc = send_controller_request(api, request, false); -+ free_xml(request); -+ return rc; -+} -+ -+// \return Standard Pacemaker return code -+static int -+controller_resource_op(pcmk_ipc_api_t *api, const char *op, -+ const char *target_node, const char *router_node, -+ bool cib_only, const char *rsc_id, -+ const char *rsc_long_id, const char *standard, -+ const char *provider, const char *type) -+{ -+ int rc = pcmk_rc_ok; -+ char *key; -+ xmlNode *request, *msg_data, *xml_rsc, *params; -+ -+ if (api == NULL) { -+ return EINVAL; -+ } -+ if (router_node == NULL) { -+ router_node = target_node; -+ } -+ -+ msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP); -+ -+ /* The controller logs the transition key from resource op requests, so we -+ * need to have *something* for it. -+ * @TODO don't use "crm-resource" -+ */ -+ key = pcmk__transition_key(0, getpid(), 0, -+ "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx"); -+ crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key); -+ free(key); -+ -+ crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node); -+ if (safe_str_neq(router_node, target_node)) { -+ crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node); -+ } -+ -+ if (cib_only) { -+ // Indicate that only the CIB needs to be cleaned -+ crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB); -+ } -+ -+ xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE); -+ crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id); -+ crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc_long_id); -+ crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, standard); -+ crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, provider); -+ crm_xml_add(xml_rsc, XML_ATTR_TYPE, type); -+ -+ params = create_xml_node(msg_data, XML_TAG_ATTRS); -+ crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET); -+ -+ // The controller parses the timeout from the request -+ key = crm_meta_name(XML_ATTR_TIMEOUT); -+ crm_xml_add(params, key, "60000"); /* 1 minute */ //@TODO pass as arg -+ free(key); -+ -+ request = create_controller_request(api, op, router_node, msg_data); -+ rc = send_controller_request(api, request, true); -+ free_xml(msg_data); -+ free_xml(request); -+ return rc; -+} -+ -+/*! -+ * \brief Ask the controller to fail a resource -+ * -+ * \param[in] api Controller connection -+ * \param[in] target_node Name of node resource is on -+ * \param[in] router_node Router node for target -+ * \param[in] rsc_id ID of resource to fail -+ * \param[in] rsc_long_id Long ID of resource (if any) -+ * \param[in] standard Standard of resource -+ * \param[in] provider Provider of resource (if any) -+ * \param[in] type Type of resource to fail -+ * -+ * \return Standard Pacemaker return code -+ * \note Event callback will get a reply of type pcmk_controld_reply_resource. -+ */ -+int -+pcmk_controld_api_fail(pcmk_ipc_api_t *api, -+ const char *target_node, const char *router_node, -+ const char *rsc_id, const char *rsc_long_id, -+ const char *standard, const char *provider, -+ const char *type) -+{ -+ crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s", -+ pcmk_ipc_name(api, true), crm_str(rsc_id), crm_str(rsc_long_id), -+ crm_str(target_node), crm_str(router_node)); -+ return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node, -+ router_node, false, rsc_id, rsc_long_id, -+ standard, provider, type); -+} -+ -+/*! -+ * \brief Ask the controller to refresh a resource -+ * -+ * \param[in] api Controller connection -+ * \param[in] target_node Name of node resource is on -+ * \param[in] router_node Router node for target -+ * \param[in] rsc_id ID of resource to refresh -+ * \param[in] rsc_long_id Long ID of resource (if any) -+ * \param[in] standard Standard of resource -+ * \param[in] provider Provider of resource (if any) -+ * \param[in] type Type of resource -+ * \param[in] cib_only If true, clean resource from CIB only -+ * -+ * \return Standard Pacemaker return code -+ * \note Event callback will get a reply of type pcmk_controld_reply_resource. -+ */ -+int -+pcmk_controld_api_refresh(pcmk_ipc_api_t *api, const char *target_node, -+ const char *router_node, -+ const char *rsc_id, const char *rsc_long_id, -+ const char *standard, const char *provider, -+ const char *type, bool cib_only) -+{ -+ crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s", -+ pcmk_ipc_name(api, true), crm_str(rsc_id), crm_str(rsc_long_id), -+ crm_str(target_node), crm_str(router_node)); -+ return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node, -+ router_node, cib_only, rsc_id, rsc_long_id, -+ standard, provider, type); -+} -+ -+/*! -+ * \brief Get the number of IPC replies currently expected from the controller -+ * -+ * \param[in] api Controller IPC API connection -+ * -+ * \return Number of replies expected -+ */ -+unsigned int -+pcmk_controld_api_replies_expected(pcmk_ipc_api_t *api) -+{ -+ struct controld_api_private_s *private = api->api_data; -+ -+ return private->replies_expected; -+} -+ -+xmlNode * -+create_hello_message(const char *uuid, const char *client_name, -+ const char *major_version, const char *minor_version) -+{ -+ xmlNode *hello_node = NULL; -+ xmlNode *hello = NULL; -+ -+ if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name) -+ || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) { -+ crm_err("Could not create IPC hello message from %s (UUID %s): " -+ "missing information", -+ client_name? client_name : "unknown client", -+ uuid? uuid : "unknown"); -+ return NULL; -+ } -+ -+ hello_node = create_xml_node(NULL, XML_TAG_OPTIONS); -+ if (hello_node == NULL) { -+ crm_err("Could not create IPC hello message from %s (UUID %s): " -+ "Message data creation failed", client_name, uuid); -+ return NULL; -+ } -+ -+ crm_xml_add(hello_node, "major_version", major_version); -+ crm_xml_add(hello_node, "minor_version", minor_version); -+ crm_xml_add(hello_node, "client_name", client_name); -+ crm_xml_add(hello_node, "client_uuid", uuid); -+ -+ hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid); -+ if (hello == NULL) { -+ crm_err("Could not create IPC hello message from %s (UUID %s): " -+ "Request creation failed", client_name, uuid); -+ return NULL; -+ } -+ free_xml(hello_node); -+ -+ crm_trace("Created hello message from %s (UUID %s)", client_name, uuid); -+ return hello; -+} --- -1.8.3.1 - - -From 1d1b34664b64f6805aeaf4d8e29e77aa8f59b4fc Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 7 Apr 2020 15:43:01 -0500 -Subject: [PATCH 3/6] Refactor: tools: convert crm_resource to use new - controller IPC model - ---- - tools/Makefile.am | 3 +- - tools/crm_resource.c | 138 +++++++------ - tools/crm_resource.h | 9 +- - tools/crm_resource_controller.c | 425 ---------------------------------------- - tools/crm_resource_controller.h | 198 ------------------- - tools/crm_resource_runtime.c | 30 +-- - 6 files changed, 96 insertions(+), 707 deletions(-) - delete mode 100644 tools/crm_resource_controller.c - delete mode 100644 tools/crm_resource_controller.h - -diff --git a/tools/Makefile.am b/tools/Makefile.am -index c822a8c..4609b0f 100644 ---- a/tools/Makefile.am -+++ b/tools/Makefile.am -@@ -12,7 +12,7 @@ if BUILD_SYSTEMD - systemdsystemunit_DATA = crm_mon.service - endif - --noinst_HEADERS = crm_mon.h crm_resource.h crm_resource_controller.h -+noinst_HEADERS = crm_mon.h crm_resource.h - - pcmkdir = $(datadir)/$(PACKAGE) - pcmk_DATA = report.common report.collector -@@ -115,7 +115,6 @@ crm_attribute_LDADD = $(top_builddir)/lib/cluster/libcrmcluster.la \ - - crm_resource_SOURCES = crm_resource.c \ - crm_resource_ban.c \ -- crm_resource_controller.c \ - crm_resource_print.c \ - crm_resource_runtime.c - crm_resource_LDADD = $(top_builddir)/lib/pengine/libpe_rules.la \ -diff --git a/tools/crm_resource.c b/tools/crm_resource.c -index 6853ad5..c8c1cfa 100644 ---- a/tools/crm_resource.c -+++ b/tools/crm_resource.c -@@ -11,62 +11,70 @@ - #include - - #include -- --#include --#include -- - #include - #include - #include -- - #include - #include - #include - #include - #include - -+#include -+#include -+#include -+ - bool BE_QUIET = FALSE; - bool scope_master = FALSE; - int cib_options = cib_sync_call; -- --static GMainLoop *mainloop = NULL; -+static crm_exit_t exit_code = CRM_EX_OK; - - // Things that should be cleaned up on exit -+static GMainLoop *mainloop = NULL; - static cib_t *cib_conn = NULL; --static pcmk_controld_api_t *controld_api = NULL; -+static pcmk_ipc_api_t *controld_api = NULL; - static pe_working_set_t *data_set = NULL; - - #define MESSAGE_TIMEOUT_S 60 - - // Clean up and exit - static crm_exit_t --bye(crm_exit_t exit_code) -+bye(crm_exit_t ec) - { -- static bool crm_resource_shutdown_flag = false; -- -- if (crm_resource_shutdown_flag) { -- // Allow usage like "return bye(...);" -- return exit_code; -- } -- crm_resource_shutdown_flag = true; -- - if (cib_conn != NULL) { - cib_t *save_cib_conn = cib_conn; - -- cib_conn = NULL; -+ cib_conn = NULL; // Ensure we can't free this twice - save_cib_conn->cmds->signoff(save_cib_conn); - cib_delete(save_cib_conn); - } - if (controld_api != NULL) { -- pcmk_controld_api_t *save_controld_api = controld_api; -+ pcmk_ipc_api_t *save_controld_api = controld_api; - -- controld_api = NULL; -- pcmk_free_controld_api(save_controld_api); -+ controld_api = NULL; // Ensure we can't free this twice -+ pcmk_free_ipc_api(save_controld_api); -+ } -+ if (mainloop != NULL) { -+ g_main_loop_unref(mainloop); -+ mainloop = NULL; - } - pe_free_working_set(data_set); - data_set = NULL; -- crm_exit(exit_code); -- return exit_code; -+ crm_exit(ec); -+ return ec; -+} -+ -+static void -+quit_main_loop(crm_exit_t ec) -+{ -+ exit_code = ec; -+ if (mainloop != NULL) { -+ GMainLoop *mloop = mainloop; -+ -+ mainloop = NULL; // Don't re-enter this block -+ pcmk_quit_main_loop(mloop, 10); -+ g_main_loop_unref(mloop); -+ } - } - - static gboolean -@@ -76,39 +84,54 @@ resource_ipc_timeout(gpointer data) - fprintf(stderr, "\nAborting because no messages received in %d seconds\n", - MESSAGE_TIMEOUT_S); - crm_err("No messages received in %d seconds", MESSAGE_TIMEOUT_S); -- bye(CRM_EX_TIMEOUT); -+ quit_main_loop(CRM_EX_TIMEOUT); - return FALSE; - } - - static void --handle_controller_reply(pcmk_controld_api_t *capi, void *api_data, -- void *user_data) -+controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type, -+ crm_exit_t status, void *event_data, void *user_data) - { -- fprintf(stderr, "."); -- if ((capi->replies_expected(capi) == 0) -- && mainloop && g_main_loop_is_running(mainloop)) { -- fprintf(stderr, " OK\n"); -- crm_debug("Got all the replies we expected"); -- bye(CRM_EX_OK); -- } --} -+ switch (event_type) { -+ case pcmk_ipc_event_disconnect: -+ if (exit_code == CRM_EX_DISCONNECT) { // Unexpected -+ crm_info("Connection to controller was terminated"); -+ } -+ quit_main_loop(exit_code); -+ break; - --static void --handle_controller_drop(pcmk_controld_api_t *capi, void *api_data, -- void *user_data) --{ -- crm_info("Connection to controller was terminated"); -- bye(CRM_EX_DISCONNECT); -+ case pcmk_ipc_event_reply: -+ if (status != CRM_EX_OK) { -+ fprintf(stderr, "\nError: bad reply from controller: %s\n", -+ crm_exit_str(status)); -+ pcmk_disconnect_ipc(api); -+ quit_main_loop(status); -+ } else { -+ fprintf(stderr, "."); -+ if ((pcmk_controld_api_replies_expected(api) == 0) -+ && mainloop && g_main_loop_is_running(mainloop)) { -+ fprintf(stderr, " OK\n"); -+ crm_debug("Got all the replies we expected"); -+ pcmk_disconnect_ipc(api); -+ quit_main_loop(CRM_EX_OK); -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } - } - - static void --start_mainloop(pcmk_controld_api_t *capi) -+start_mainloop(pcmk_ipc_api_t *capi) - { -- if (capi->replies_expected(capi) > 0) { -- unsigned int count = capi->replies_expected(capi); -+ unsigned int count = pcmk_controld_api_replies_expected(capi); - -+ if (count > 0) { - fprintf(stderr, "Waiting for %d %s from the controller", - count, pcmk__plural_alt(count, "reply", "replies")); -+ exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects - mainloop = g_main_loop_new(NULL, FALSE); - g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL); - g_main_loop_run(mainloop); -@@ -664,7 +687,6 @@ main(int argc, char **argv) - int argerr = 0; - int flag; - int find_flags = 0; // Flags to use when searching for resource -- crm_exit_t exit_code = CRM_EX_OK; - - crm_log_cli_init("crm_resource"); - pcmk__set_cli_options(NULL, "| [options]", long_options, -@@ -1151,21 +1173,15 @@ main(int argc, char **argv) - - // Establish a connection to the controller if needed - if (require_crmd) { -- char *client_uuid; -- pcmk_controld_api_cb_t dispatch_cb = { -- handle_controller_reply, NULL -- }; -- pcmk_controld_api_cb_t destroy_cb = { -- handle_controller_drop, NULL -- }; -- -- -- client_uuid = pcmk__getpid_s(); -- controld_api = pcmk_new_controld_api(crm_system_name, client_uuid); -- free(client_uuid); -- -- rc = controld_api->connect(controld_api, true, &dispatch_cb, -- &destroy_cb); -+ rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld); -+ if (rc != pcmk_rc_ok) { -+ CMD_ERR("Error connecting to the controller: %s", pcmk_rc_str(rc)); -+ rc = pcmk_rc2legacy(rc); -+ goto bail; -+ } -+ pcmk_register_ipc_callback(controld_api, controller_event_callback, -+ NULL); -+ rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main); - if (rc != pcmk_rc_ok) { - CMD_ERR("Error connecting to the controller: %s", pcmk_rc_str(rc)); - rc = pcmk_rc2legacy(rc); -@@ -1525,8 +1541,8 @@ main(int argc, char **argv) - NULL, NULL, NULL, - NULL, attr_options)); - -- if (controld_api->reprobe(controld_api, host_uname, -- router_node) == pcmk_rc_ok) { -+ if (pcmk_controld_api_reprobe(controld_api, host_uname, -+ router_node) == pcmk_rc_ok) { - start_mainloop(controld_api); - } - -diff --git a/tools/crm_resource.h b/tools/crm_resource.h -index 0bf7bee..cb7f506 100644 ---- a/tools/crm_resource.h -+++ b/tools/crm_resource.h -@@ -21,7 +21,6 @@ - #include - #include - #include --#include "crm_resource_controller.h" - - extern bool print_pending; - -@@ -36,8 +35,6 @@ extern char *move_lifetime; - - extern const char *attr_set_type; - --extern pcmk_controld_api_cb_t controld_api_cb; -- - /* ban */ - int cli_resource_prefer(const char *rsc_id, const char *host, cib_t * cib_conn); - int cli_resource_ban(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn); -@@ -63,16 +60,16 @@ int cli_resource_print_operations(const char *rsc_id, const char *host_uname, bo - - /* runtime */ - void cli_resource_check(cib_t * cib, pe_resource_t *rsc); --int cli_resource_fail(pcmk_controld_api_t *controld_api, -+int cli_resource_fail(pcmk_ipc_api_t *controld_api, - const char *host_uname, const char *rsc_id, - pe_working_set_t *data_set); - int cli_resource_search(pe_resource_t *rsc, const char *requested_name, - pe_working_set_t *data_set); --int cli_resource_delete(pcmk_controld_api_t *controld_api, -+int cli_resource_delete(pcmk_ipc_api_t *controld_api, - const char *host_uname, pe_resource_t *rsc, - const char *operation, const char *interval_spec, - bool just_failures, pe_working_set_t *data_set); --int cli_cleanup_all(pcmk_controld_api_t *controld_api, const char *node_name, -+int cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name, - const char *operation, const char *interval_spec, - pe_working_set_t *data_set); - int cli_resource_restart(pe_resource_t *rsc, const char *host, int timeout_ms, -diff --git a/tools/crm_resource_controller.c b/tools/crm_resource_controller.c -deleted file mode 100644 -index 994a7be..0000000 ---- a/tools/crm_resource_controller.c -+++ /dev/null -@@ -1,425 +0,0 @@ --/* -- * Copyright 2020 the Pacemaker project contributors -- * -- * The version control history for this file may have further details. -- * -- * This source code is licensed under the GNU General Public License version 2 -- * or later (GPLv2+) WITHOUT ANY WARRANTY. -- */ -- --#include --#include --#include --#include "crm_resource.h" -- --// API object's private members --struct controller_private { -- char *client_name; // Client name to use with IPC -- char *client_uuid; // Client UUID to use with IPC -- mainloop_io_t *source; // If main loop used, I/O source for IPC -- crm_ipc_t *ipc; // IPC connection to controller -- int replies_expected; // How many controller replies are expected -- pcmk_controld_api_cb_t dispatch_cb; // Caller's registered dispatch callback -- pcmk_controld_api_cb_t destroy_cb; // Caller's registered destroy callback --}; -- --static void --call_client_callback(pcmk_controld_api_t *api, pcmk_controld_api_cb_t *cb, -- void *api_data) --{ -- if ((cb != NULL) && (cb->callback != NULL)) { -- cb->callback(api, api_data, cb->user_data); -- } --} -- --/* -- * IPC callbacks when used with main loop -- */ -- --static void --controller_ipc_destroy(gpointer user_data) --{ -- pcmk_controld_api_t *api = user_data; -- struct controller_private *private = api->private; -- -- private->ipc = NULL; -- private->source = NULL; -- call_client_callback(api, &(private->destroy_cb), NULL); --} -- --// \return < 0 if connection is no longer required, >= 0 if it is --static int --controller_ipc_dispatch(const char *buffer, ssize_t length, gpointer user_data) --{ -- xmlNode *msg = NULL; -- pcmk_controld_api_t *api = user_data; -- -- CRM_CHECK(buffer && api && api->private, return 0); -- -- msg = string2xml(buffer); -- if (msg == NULL) { -- crm_warn("Received malformed controller IPC message"); -- } else { -- struct controller_private *private = api->private; -- -- crm_log_xml_trace(msg, "controller-reply"); -- private->replies_expected--; -- call_client_callback(api, &(private->dispatch_cb), -- get_message_xml(msg, F_CRM_DATA)); -- free_xml(msg); -- } -- return 0; --} -- --/* -- * IPC utilities -- */ -- --// \return Standard Pacemaker return code --static int --send_hello(crm_ipc_t *ipc, const char *client_name, const char *client_uuid) --{ -- xmlNode *hello = create_hello_message(client_uuid, client_name, "0", "1"); -- int rc = crm_ipc_send(ipc, hello, 0, 0, NULL); -- -- free_xml(hello); -- if (rc < 0) { -- rc = pcmk_legacy2rc(rc); -- crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s", -- CRM_SYSTEM_CRMD /* ipc->name */, -- pcmk_rc_str(rc), rc); -- return rc; -- } -- crm_debug("Sent IPC hello to %s", CRM_SYSTEM_CRMD /* ipc->name */); -- return pcmk_rc_ok; --} -- --// \return Standard Pacemaker return code --static int --send_controller_request(pcmk_controld_api_t *api, const char *op, -- xmlNode *msg_data, const char *node) --{ -- int rc; -- struct controller_private *private = api->private; -- xmlNode *cmd = create_request(op, msg_data, node, CRM_SYSTEM_CRMD, -- private->client_name, private->client_uuid); -- const char *reference = crm_element_value(cmd, XML_ATTR_REFERENCE); -- -- if ((cmd == NULL) || (reference == NULL)) { -- return EINVAL; -- } -- -- //@TODO pass as args? 0=crm_ipc_flags, 0=timeout_ms (default 5s), NULL=reply -- crm_log_xml_trace(cmd, "controller-request"); -- rc = crm_ipc_send(private->ipc, cmd, 0, 0, NULL); -- free_xml(cmd); -- if (rc < 0) { -- return pcmk_legacy2rc(rc); -- } -- private->replies_expected++; -- return pcmk_rc_ok; --} -- --/* -- * pcmk_controld_api_t methods -- */ -- --static int --controller_connect_mainloop(pcmk_controld_api_t *api) --{ -- struct controller_private *private = api->private; -- struct ipc_client_callbacks callbacks = { -- .dispatch = controller_ipc_dispatch, -- .destroy = controller_ipc_destroy, -- }; -- -- private->source = mainloop_add_ipc_client(CRM_SYSTEM_CRMD, -- G_PRIORITY_DEFAULT, 0, api, -- &callbacks); -- if (private->source == NULL) { -- return ENOTCONN; -- } -- -- private->ipc = mainloop_get_ipc_client(private->source); -- if (private->ipc == NULL) { -- (void) api->disconnect(api); -- return ENOTCONN; -- } -- -- crm_debug("Connected to %s IPC (attaching to main loop)", CRM_SYSTEM_CRMD); -- return pcmk_rc_ok; --} -- --static int --controller_connect_no_mainloop(pcmk_controld_api_t *api) --{ -- struct controller_private *private = api->private; -- -- private->ipc = crm_ipc_new(CRM_SYSTEM_CRMD, 0); -- if (private->ipc == NULL) { -- return ENOTCONN; -- } -- if (!crm_ipc_connect(private->ipc)) { -- crm_ipc_close(private->ipc); -- crm_ipc_destroy(private->ipc); -- private->ipc = NULL; -- return errno; -- } -- /* @TODO caller needs crm_ipc_get_fd(private->ipc); either add method for -- * that, or replace use_mainloop with int *fd -- */ -- crm_debug("Connected to %s IPC", CRM_SYSTEM_CRMD); -- return pcmk_rc_ok; --} -- --static void --set_callback(pcmk_controld_api_cb_t *dest, pcmk_controld_api_cb_t *source) --{ -- if (source) { -- dest->callback = source->callback; -- dest->user_data = source->user_data; -- } --} -- --static int --controller_api_connect(pcmk_controld_api_t *api, bool use_mainloop, -- pcmk_controld_api_cb_t *dispatch_cb, -- pcmk_controld_api_cb_t *destroy_cb) --{ -- int rc = pcmk_rc_ok; -- struct controller_private *private; -- -- if (api == NULL) { -- return EINVAL; -- } -- private = api->private; -- -- set_callback(&(private->dispatch_cb), dispatch_cb); -- set_callback(&(private->destroy_cb), destroy_cb); -- -- if (private->ipc != NULL) { -- return pcmk_rc_ok; // already connected -- } -- -- if (use_mainloop) { -- rc = controller_connect_mainloop(api); -- } else { -- rc = controller_connect_no_mainloop(api); -- } -- if (rc != pcmk_rc_ok) { -- return rc; -- } -- -- rc = send_hello(private->ipc, private->client_name, private->client_uuid); -- if (rc != pcmk_rc_ok) { -- (void) api->disconnect(api); -- } -- return rc; --} -- --static int --controller_api_disconnect(pcmk_controld_api_t *api) --{ -- struct controller_private *private = api->private; -- -- if (private->source != NULL) { -- // Attached to main loop -- mainloop_del_ipc_client(private->source); -- private->source = NULL; -- private->ipc = NULL; -- -- } else if (private->ipc != NULL) { -- // Not attached to main loop -- crm_ipc_t *ipc = private->ipc; -- -- private->ipc = NULL; -- crm_ipc_close(ipc); -- crm_ipc_destroy(ipc); -- } -- crm_debug("Disconnected from %s IPC", CRM_SYSTEM_CRMD /* ipc->name */); -- return pcmk_rc_ok; --} -- --//@TODO dispatch function for non-mainloop a la stonith_dispatch() --//@TODO convenience retry-connect function a la stonith_api_connect_retry() -- --static unsigned int --controller_api_replies_expected(pcmk_controld_api_t *api) --{ -- if (api != NULL) { -- struct controller_private *private = api->private; -- -- return private->replies_expected; -- } -- return 0; --} -- --static xmlNode * --create_reprobe_message_data(const char *target_node, const char *router_node) --{ -- xmlNode *msg_data; -- -- msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE); -- crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node); -- if ((router_node != NULL) && safe_str_neq(router_node, target_node)) { -- crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node); -- } -- return msg_data; --} -- --static int --controller_api_reprobe(pcmk_controld_api_t *api, const char *target_node, -- const char *router_node) --{ -- int rc = EINVAL; -- -- if (api != NULL) { -- xmlNode *msg_data; -- -- crm_debug("Sending %s IPC request to reprobe %s via %s", -- CRM_SYSTEM_CRMD, crm_str(target_node), crm_str(router_node)); -- msg_data = create_reprobe_message_data(target_node, router_node); -- rc = send_controller_request(api, CRM_OP_REPROBE, msg_data, -- (router_node? router_node : target_node)); -- free_xml(msg_data); -- } -- return rc; --} -- --// \return Standard Pacemaker return code --static int --controller_resource_op(pcmk_controld_api_t *api, const char *op, -- const char *target_node, const char *router_node, -- bool cib_only, const char *rsc_id, -- const char *rsc_long_id, const char *standard, -- const char *provider, const char *type) --{ -- int rc; -- char *key; -- xmlNode *msg_data, *xml_rsc, *params; -- -- if (api == NULL) { -- return EINVAL; -- } -- if (router_node == NULL) { -- router_node = target_node; -- } -- -- msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP); -- -- /* The controller logs the transition key from resource op requests, so we -- * need to have *something* for it. -- */ -- key = pcmk__transition_key(0, getpid(), 0, -- "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx"); -- crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key); -- free(key); -- -- crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node); -- if (safe_str_neq(router_node, target_node)) { -- crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node); -- } -- -- if (cib_only) { -- // Indicate that only the CIB needs to be cleaned -- crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB); -- } -- -- xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE); -- crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id); -- crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc_long_id); -- crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, standard); -- crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, provider); -- crm_xml_add(xml_rsc, XML_ATTR_TYPE, type); -- -- params = create_xml_node(msg_data, XML_TAG_ATTRS); -- crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET); -- -- // The controller parses the timeout from the request -- key = crm_meta_name(XML_ATTR_TIMEOUT); -- crm_xml_add(params, key, "60000"); /* 1 minute */ //@TODO pass as arg -- free(key); -- -- rc = send_controller_request(api, op, msg_data, router_node); -- free_xml(msg_data); -- return rc; --} -- --static int --controller_api_fail_resource(pcmk_controld_api_t *api, -- const char *target_node, const char *router_node, -- const char *rsc_id, const char *rsc_long_id, -- const char *standard, const char *provider, -- const char *type) --{ -- crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s", -- CRM_SYSTEM_CRMD, crm_str(rsc_id), crm_str(rsc_long_id), -- crm_str(target_node), crm_str(router_node)); -- return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node, -- router_node, false, rsc_id, rsc_long_id, -- standard, provider, type); --} -- --static int --controller_api_refresh_resource(pcmk_controld_api_t *api, -- const char *target_node, -- const char *router_node, -- const char *rsc_id, const char *rsc_long_id, -- const char *standard, const char *provider, -- const char *type, bool cib_only) --{ -- crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s", -- CRM_SYSTEM_CRMD, crm_str(rsc_id), crm_str(rsc_long_id), -- crm_str(target_node), crm_str(router_node)); -- return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node, -- router_node, cib_only, rsc_id, rsc_long_id, -- standard, provider, type); --} -- --pcmk_controld_api_t * --pcmk_new_controld_api(const char *client_name, const char *client_uuid) --{ -- struct controller_private *private; -- pcmk_controld_api_t *api = calloc(1, sizeof(pcmk_controld_api_t)); -- -- CRM_ASSERT(api != NULL); -- -- api->private = calloc(1, sizeof(struct controller_private)); -- CRM_ASSERT(api->private != NULL); -- private = api->private; -- -- if (client_name == NULL) { -- client_name = crm_system_name? crm_system_name : "client"; -- } -- private->client_name = strdup(client_name); -- CRM_ASSERT(private->client_name != NULL); -- -- if (client_uuid == NULL) { -- private->client_uuid = crm_generate_uuid(); -- } else { -- private->client_uuid = strdup(client_uuid); -- } -- CRM_ASSERT(private->client_uuid != NULL); -- -- api->connect = controller_api_connect; -- api->disconnect = controller_api_disconnect; -- api->replies_expected = controller_api_replies_expected; -- api->reprobe = controller_api_reprobe; -- api->fail_resource = controller_api_fail_resource; -- api->refresh_resource = controller_api_refresh_resource; -- return api; --} -- --void --pcmk_free_controld_api(pcmk_controld_api_t *api) --{ -- if (api != NULL) { -- struct controller_private *private = api->private; -- -- api->disconnect(api); -- free(private->client_name); -- free(private->client_uuid); -- free(api->private); -- free(api); -- } --} -diff --git a/tools/crm_resource_controller.h b/tools/crm_resource_controller.h -deleted file mode 100644 -index 50e20b4..0000000 ---- a/tools/crm_resource_controller.h -+++ /dev/null -@@ -1,198 +0,0 @@ --/* -- * Copyright 2020 the Pacemaker project contributors -- * -- * The version control history for this file may have further details. -- * -- * This source code is licensed under the GNU Lesser General Public License -- * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -- */ --#ifndef PCMK__CONTROLD_API_H --#define PCMK__CONTROLD_API_H -- --#ifdef __cplusplus --extern "C" { --#endif -- --#include // bool -- --/* This is a demonstration of an abstracted controller IPC API. It is expected -- * that this will be improved and moved to libcrmcommon. -- * -- * @TODO We could consider whether it's reasonable to have a single type for -- * all daemons' IPC APIs (e.g. pcmk_ipc_api_t instead of pcmk_*_api_t). They -- * could potentially have common connect/disconnect methods and then a void* to -- * a group of API-specific methods. -- * -- * In that case, the callback type would also need to be generic, taking -- * (pcmk_ipc_api_t *api, void *api_data, void *user_data), with individual APIs -- * having functions for getting useful info from api_data. If all APIs followed -- * the call_id model, we could use int call_id instead of api_data. -- * -- * A major annoyance is that the controller IPC protocol currently does not have -- * any way to tie a particular reply to a particular request. The current -- * clients (crmadmin, crm_node, and crm_resource) simply know what kind of reply -- * to expect for the kind of request they sent. In crm_resource's case, all it -- * does is count replies, ignoring their content altogether. -- * -- * That really forces us to have a single callback for all events rather than a -- * per-request callback. That in turn implies that callers can only provide a -- * single user data pointer. -- * -- * @TODO Define protocol version constants to use in hello message. -- * @TODO Allow callers to specify timeouts. -- * @TODO Define call IDs for controller ops, while somehow maintaining backward -- * compatibility, since a client running on a Pacemaker Remote node could -- * be older or newer than the controller on the connection's cluster -- * node. -- * @TODO The controller currently does not respond to hello messages. We should -- * establish a common connection handshake protocol for all daemons that -- * involves a hello message and acknowledgement. We should support sync -- * or async connection (i.e. block until the ack is received, or return -- * after the hello is sent and call a connection callback when the hello -- * ack is received). -- */ -- --//! \internal --typedef struct pcmk_controld_api_s pcmk_controld_api_t; -- --//! \internal --typedef struct pcmk_controld_api_callback_s { -- void (*callback)(pcmk_controld_api_t *api, void *api_data, void *user_data); -- void *user_data; --} pcmk_controld_api_cb_t; -- --//! \internal --struct pcmk_controld_api_s { -- //! \internal -- void *private; -- -- /*! -- * \internal -- * \brief Connect to the local controller -- * -- * \param[in] api Controller API instance -- * \param[in] use_mainloop If true, attach IPC to main loop -- * \param[in] dispatch_cb If not NULL, call this when replies are received -- * \param[in] destroy_cb If not NULL, call this if connection drops -- * -- * \return Standard Pacemaker return code -- * \note Only the pointers inside the callback objects need to be -- * persistent, not the callback objects themselves. The destroy_cb -- * will be called only for unrequested disconnects. -- */ -- int (*connect)(pcmk_controld_api_t *api, bool use_mainloop, -- pcmk_controld_api_cb_t *dispatch_cb, -- pcmk_controld_api_cb_t *destroy_cb); -- -- /*! -- * \internal -- * \brief Disconnect from the local controller -- * -- * \param[in] api Controller API instance -- * -- * \return Standard Pacemaker return code -- */ -- int (*disconnect)(pcmk_controld_api_t *api); -- -- /*! -- * \internal -- * \brief Check number of replies still expected from controller -- * -- * \param[in] api Controller API instance -- * -- * \return Number of expected replies -- */ -- unsigned int (*replies_expected)(pcmk_controld_api_t *api); -- -- /*! -- * \internal -- * \brief Send a reprobe controller operation -- * -- * \param[in] api Controller API instance -- * \param[in] target_node Name of node to reprobe -- * \param[in] router_node Router node for host -- * -- * \return Standard Pacemaker return code -- */ -- int (*reprobe)(pcmk_controld_api_t *api, const char *target_node, -- const char *router_node); -- -- /* @TODO These methods have a lot of arguments. One possibility would be to -- * make a struct for agent info (standard/provider/type), which theortically -- * could be used throughout pacemaker code. However that would end up being -- * really awkward to use generically, since sometimes you need to allocate -- * those strings (char *) and other times you only have references into XML -- * (const char *). We could make some structs just for this API. -- */ -- -- /*! -- * \internal -- * \brief Ask the controller to fail a resource -- * -- * \param[in] api Controller API instance -- * \param[in] target_node Name of node resource is on -- * \param[in] router_node Router node for target -- * \param[in] rsc_id ID of resource to fail -- * \param[in] rsc_long_id Long ID of resource (if any) -- * \param[in] standard Standard of resource -- * \param[in] provider Provider of resource (if any) -- * \param[in] type Type of resource to fail -- * -- * \return Standard Pacemaker return code -- */ -- int (*fail_resource)(pcmk_controld_api_t *api, const char *target_node, -- const char *router_node, const char *rsc_id, -- const char *rsc_long_id, const char *standard, -- const char *provider, const char *type); -- -- /*! -- * \internal -- * \brief Ask the controller to refresh a resource -- * -- * \param[in] api Controller API instance -- * \param[in] target_node Name of node resource is on -- * \param[in] router_node Router node for target -- * \param[in] rsc_id ID of resource to refresh -- * \param[in] rsc_long_id Long ID of resource (if any) -- * \param[in] standard Standard of resource -- * \param[in] provider Provider of resource (if any) -- * \param[in] type Type of resource -- * \param[in] cib_only If true, clean resource from CIB only -- * -- * \return Standard Pacemaker return code -- */ -- int (*refresh_resource)(pcmk_controld_api_t *api, const char *target_node, -- const char *router_node, const char *rsc_id, -- const char *rsc_long_id, const char *standard, -- const char *provider, const char *type, -- bool cib_only); --}; -- --/*! -- * \internal -- * \brief Create new controller IPC API object for clients -- * -- * \param[in] client_name Client name to use with IPC -- * \param[in] client_uuid Client UUID to use with IPC -- * -- * \return Newly allocated object -- * \note This function asserts on errors, so it will never return NULL. -- * The caller is responsible for freeing the result with -- * pcmk_free_controld_api(). -- */ --pcmk_controld_api_t *pcmk_new_controld_api(const char *client_name, -- const char *client_uuid); -- --/*! -- * \internal -- * \brief Free a controller IPC API object -- * -- * \param[in] api Controller IPC API object to free -- */ --void pcmk_free_controld_api(pcmk_controld_api_t *api); -- --#ifdef __cplusplus --} --#endif -- --#endif -diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c -index 37789d1..cc2abeb 100644 ---- a/tools/crm_resource_runtime.c -+++ b/tools/crm_resource_runtime.c -@@ -8,6 +8,7 @@ - */ - - #include -+#include - - int resource_verbose = 0; - bool do_force = FALSE; -@@ -460,7 +461,7 @@ cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name, - - // \return Standard Pacemaker return code - static int --send_lrm_rsc_op(pcmk_controld_api_t *controld_api, bool do_fail_resource, -+send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource, - const char *host_uname, const char *rsc_id, - pe_working_set_t *data_set) - { -@@ -528,14 +529,13 @@ send_lrm_rsc_op(pcmk_controld_api_t *controld_api, bool do_fail_resource, - rsc_api_id = rsc->id; - } - if (do_fail_resource) { -- return controld_api->fail_resource(controld_api, host_uname, -- router_node, rsc_api_id, rsc_long_id, -- rsc_class, rsc_provider, rsc_type); -+ return pcmk_controld_api_fail(controld_api, host_uname, router_node, -+ rsc_api_id, rsc_long_id, -+ rsc_class, rsc_provider, rsc_type); - } else { -- return controld_api->refresh_resource(controld_api, host_uname, -- router_node, rsc_api_id, -- rsc_long_id, rsc_class, -- rsc_provider, rsc_type, cib_only); -+ return pcmk_controld_api_refresh(controld_api, host_uname, router_node, -+ rsc_api_id, rsc_long_id, rsc_class, -+ rsc_provider, rsc_type, cib_only); - } - } - -@@ -558,7 +558,7 @@ rsc_fail_name(pe_resource_t *rsc) - - // \return Standard Pacemaker return code - static int --clear_rsc_history(pcmk_controld_api_t *controld_api, const char *host_uname, -+clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname, - const char *rsc_id, pe_working_set_t *data_set) - { - int rc = pcmk_ok; -@@ -574,16 +574,16 @@ clear_rsc_history(pcmk_controld_api_t *controld_api, const char *host_uname, - } - - crm_trace("Processing %d mainloop inputs", -- controld_api->replies_expected(controld_api)); -+ pcmk_controld_api_replies_expected(controld_api)); - while (g_main_context_iteration(NULL, FALSE)) { - crm_trace("Processed mainloop input, %d still remaining", -- controld_api->replies_expected(controld_api)); -+ pcmk_controld_api_replies_expected(controld_api)); - } - return rc; - } - - static int --clear_rsc_failures(pcmk_controld_api_t *controld_api, const char *node_name, -+clear_rsc_failures(pcmk_ipc_api_t *controld_api, const char *node_name, - const char *rsc_id, const char *operation, - const char *interval_spec, pe_working_set_t *data_set) - { -@@ -683,7 +683,7 @@ clear_rsc_fail_attrs(pe_resource_t *rsc, const char *operation, - } - - int --cli_resource_delete(pcmk_controld_api_t *controld_api, const char *host_uname, -+cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname, - pe_resource_t *rsc, const char *operation, - const char *interval_spec, bool just_failures, - pe_working_set_t *data_set) -@@ -792,7 +792,7 @@ cli_resource_delete(pcmk_controld_api_t *controld_api, const char *host_uname, - } - - int --cli_cleanup_all(pcmk_controld_api_t *controld_api, const char *node_name, -+cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name, - const char *operation, const char *interval_spec, - pe_working_set_t *data_set) - { -@@ -905,7 +905,7 @@ cli_resource_check(cib_t * cib_conn, pe_resource_t *rsc) - - // \return Standard Pacemaker return code - int --cli_resource_fail(pcmk_controld_api_t *controld_api, const char *host_uname, -+cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname, - const char *rsc_id, pe_working_set_t *data_set) - { - crm_notice("Failing %s on %s", rsc_id, host_uname); --- -1.8.3.1 - - -From ae14fa4a831e45eae0d78b0f42765fcf40c4ce56 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 14 Apr 2020 14:06:02 -0500 -Subject: [PATCH 4/6] Refactor: tools: convert crm_node to use new controller - IPC model - ---- - tools/crm_node.c | 281 +++++++++++++++++++++++++++---------------------------- - 1 file changed, 140 insertions(+), 141 deletions(-) - -diff --git a/tools/crm_node.c b/tools/crm_node.c -index db31f20..1773a36 100644 ---- a/tools/crm_node.c -+++ b/tools/crm_node.c -@@ -19,6 +19,7 @@ - #include - #include - #include -+#include - #include - - #define SUMMARY "crm_node - Tool for displaying low-level node information" -@@ -39,7 +40,6 @@ gboolean command_cb(const gchar *option_name, const gchar *optarg, gpointer data - gboolean name_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error); - gboolean remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error); - --static char *pid_s = NULL; - static GMainLoop *mainloop = NULL; - static crm_exit_t exit_code = CRM_EX_OK; - -@@ -140,11 +140,6 @@ remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError * - static void - crm_node_exit(crm_exit_t value) - { -- if (pid_s) { -- free(pid_s); -- pid_s = NULL; -- } -- - exit_code = value; - - if (mainloop && g_main_loop_is_running(mainloop)) { -@@ -155,173 +150,123 @@ crm_node_exit(crm_exit_t value) - } - - static void --exit_disconnect(gpointer user_data) --{ -- fprintf(stderr, "error: Lost connection to cluster\n"); -- crm_node_exit(CRM_EX_DISCONNECT); --} -- --typedef int (*ipc_dispatch_fn) (const char *buffer, ssize_t length, -- gpointer userdata); -- --static crm_ipc_t * --new_mainloop_for_ipc(const char *system, ipc_dispatch_fn dispatch) -+controller_event_cb(pcmk_ipc_api_t *controld_api, -+ enum pcmk_ipc_event event_type, crm_exit_t status, -+ void *event_data, void *user_data) - { -- mainloop_io_t *source = NULL; -- crm_ipc_t *ipc = NULL; -- -- struct ipc_client_callbacks ipc_callbacks = { -- .dispatch = dispatch, -- .destroy = exit_disconnect -- }; -+ pcmk_controld_api_reply_t *reply = event_data; - -- mainloop = g_main_loop_new(NULL, FALSE); -- source = mainloop_add_ipc_client(system, G_PRIORITY_DEFAULT, 0, -- NULL, &ipc_callbacks); -- ipc = mainloop_get_ipc_client(source); -- if (ipc == NULL) { -- fprintf(stderr, -- "error: Could not connect to cluster (is it running?)\n"); -- crm_node_exit(CRM_EX_DISCONNECT); -- } -- return ipc; --} -- --static int --send_controller_hello(crm_ipc_t *controller) --{ -- xmlNode *hello = NULL; -- int rc; -- -- pid_s = pcmk__getpid_s(); -- hello = create_hello_message(pid_s, crm_system_name, "1", "0"); -- rc = crm_ipc_send(controller, hello, 0, 0, NULL); -- free_xml(hello); -- return (rc < 0)? rc : 0; --} -- --static int --send_node_info_request(crm_ipc_t *controller, uint32_t nodeid) --{ -- xmlNode *ping = NULL; -- int rc; -- -- ping = create_request(CRM_OP_NODE_INFO, NULL, NULL, CRM_SYSTEM_CRMD, -- crm_system_name, pid_s); -- if (nodeid > 0) { -- crm_xml_add_int(ping, XML_ATTR_ID, nodeid); -- } -- rc = crm_ipc_send(controller, ping, 0, 0, NULL); -- free_xml(ping); -- return (rc < 0)? rc : 0; --} -+ switch (event_type) { -+ case pcmk_ipc_event_disconnect: -+ if (exit_code == CRM_EX_DISCONNECT) { // Unexpected -+ fprintf(stderr, "error: Lost connection to controller\n"); -+ } -+ goto done; -+ break; - --static int --dispatch_controller(const char *buffer, ssize_t length, gpointer userdata) --{ -- xmlNode *message = string2xml(buffer); -- xmlNode *data = NULL; -- const char *value = NULL; -+ case pcmk_ipc_event_reply: -+ break; - -- if (message == NULL) { -- fprintf(stderr, "error: Could not understand reply from controller\n"); -- crm_node_exit(CRM_EX_PROTOCOL); -- return 0; -+ default: -+ return; - } -- crm_log_xml_trace(message, "controller reply"); -- -- exit_code = CRM_EX_PROTOCOL; - -- // Validate reply -- value = crm_element_value(message, F_CRM_MSG_TYPE); -- if (safe_str_neq(value, XML_ATTR_RESPONSE)) { -- fprintf(stderr, "error: Message from controller was not a reply\n"); -+ if (status != CRM_EX_OK) { -+ fprintf(stderr, "error: Bad reply from controller: %s\n", -+ crm_exit_str(status)); - goto done; - } -- value = crm_element_value(message, XML_ATTR_REFERENCE); -- if (value == NULL) { -- fprintf(stderr, "error: Controller reply did not specify original message\n"); -- goto done; -- } -- data = get_message_xml(message, F_CRM_DATA); -- if (data == NULL) { -- fprintf(stderr, "error: Controller reply did not contain any data\n"); -+ if (reply->reply_type != pcmk_controld_reply_info) { -+ fprintf(stderr, "error: Unknown reply type %d from controller\n", -+ reply->reply_type); - goto done; - } - -+ // Parse desired info from reply and display to user - switch (options.command) { - case 'i': -- value = crm_element_value(data, XML_ATTR_ID); -- if (value == NULL) { -- fprintf(stderr, "error: Controller reply did not contain node ID\n"); -- } else { -- printf("%s\n", value); -- exit_code = CRM_EX_OK; -+ if (reply->data.node_info.id == 0) { -+ fprintf(stderr, -+ "error: Controller reply did not contain node ID\n"); -+ exit_code = CRM_EX_PROTOCOL; -+ goto done; - } -+ printf("%d\n", reply->data.node_info.id); - break; - - case 'n': - case 'N': -- value = crm_element_value(data, XML_ATTR_UNAME); -- if (value == NULL) { -+ if (reply->data.node_info.uname == NULL) { - fprintf(stderr, "Node is not known to cluster\n"); - exit_code = CRM_EX_NOHOST; -- } else { -- printf("%s\n", value); -- exit_code = CRM_EX_OK; -+ goto done; - } -+ printf("%s\n", reply->data.node_info.uname); - break; - - case 'q': -- value = crm_element_value(data, XML_ATTR_HAVE_QUORUM); -- if (value == NULL) { -- fprintf(stderr, "error: Controller reply did not contain quorum status\n"); -- } else { -- bool quorum = crm_is_true(value); -- -- printf("%d\n", quorum); -- exit_code = quorum? CRM_EX_OK : CRM_EX_QUORUM; -+ printf("%d\n", reply->data.node_info.have_quorum); -+ if (!(reply->data.node_info.have_quorum)) { -+ exit_code = CRM_EX_QUORUM; -+ goto done; - } - break; - - default: - fprintf(stderr, "internal error: Controller reply not expected\n"); - exit_code = CRM_EX_SOFTWARE; -- break; -+ goto done; - } - -+ // Success -+ exit_code = CRM_EX_OK; - done: -- free_xml(message); -- crm_node_exit(exit_code); -- return 0; -+ pcmk_disconnect_ipc(controld_api); -+ pcmk_quit_main_loop(mainloop, 10); - } - - static void - run_controller_mainloop(uint32_t nodeid) - { -- crm_ipc_t *controller = NULL; -+ pcmk_ipc_api_t *controld_api = NULL; - int rc; - -- controller = new_mainloop_for_ipc(CRM_SYSTEM_CRMD, dispatch_controller); -+ // Set disconnect exit code to handle unexpected disconnects -+ exit_code = CRM_EX_DISCONNECT; -+ -+ // Create controller IPC object -+ rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld); -+ if (rc != pcmk_rc_ok) { -+ fprintf(stderr, "error: Could not connect to controller: %s\n", -+ pcmk_rc_str(rc)); -+ return; -+ } -+ pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL); - -- rc = send_controller_hello(controller); -- if (rc < 0) { -- fprintf(stderr, "error: Could not register with controller: %s\n", -- pcmk_strerror(rc)); -- crm_node_exit(crm_errno2exit(rc)); -+ // Connect to controller -+ rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main); -+ if (rc != pcmk_rc_ok) { -+ fprintf(stderr, "error: Could not connect to controller: %s\n", -+ pcmk_rc_str(rc)); -+ exit_code = pcmk_rc2exitc(rc); -+ return; - } - -- rc = send_node_info_request(controller, nodeid); -- if (rc < 0) { -+ rc = pcmk_controld_api_node_info(controld_api, nodeid); -+ if (rc != pcmk_rc_ok) { - fprintf(stderr, "error: Could not ping controller: %s\n", -- pcmk_strerror(rc)); -- crm_node_exit(crm_errno2exit(rc)); -+ pcmk_rc_str(rc)); -+ pcmk_disconnect_ipc(controld_api); -+ exit_code = pcmk_rc2exitc(rc); -+ return; - } - -- // Run main loop to get controller reply via dispatch_controller() -+ // Run main loop to get controller reply via controller_event_cb() -+ mainloop = g_main_loop_new(NULL, FALSE); - g_main_loop_run(mainloop); - g_main_loop_unref(mainloop); - mainloop = NULL; -+ pcmk_free_ipc_api(controld_api); - } - - static void -@@ -385,32 +330,56 @@ cib_remove_node(long id, const char *name) - } - - static int -+controller_remove_node(const char *node_name, long nodeid) -+{ -+ pcmk_ipc_api_t *controld_api = NULL; -+ int rc; -+ -+ // Create controller IPC object -+ rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld); -+ if (rc != pcmk_rc_ok) { -+ fprintf(stderr, "error: Could not connect to controller: %s\n", -+ pcmk_rc_str(rc)); -+ return ENOTCONN; -+ } -+ -+ // Connect to controller (without main loop) -+ rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_sync); -+ if (rc != pcmk_rc_ok) { -+ fprintf(stderr, "error: Could not connect to controller: %s\n", -+ pcmk_rc_str(rc)); -+ pcmk_free_ipc_api(controld_api); -+ return rc; -+ } -+ -+ rc = pcmk_ipc_purge_node(controld_api, node_name, nodeid); -+ if (rc != pcmk_rc_ok) { -+ fprintf(stderr, -+ "error: Could not clear node from controller's cache: %s\n", -+ pcmk_rc_str(rc)); -+ } -+ -+ pcmk_free_ipc_api(controld_api); -+ return pcmk_rc_ok; -+} -+ -+static int - tools_remove_node_cache(const char *node_name, long nodeid, const char *target) - { - int rc = -1; -- crm_ipc_t *conn = crm_ipc_new(target, 0); -+ crm_ipc_t *conn = NULL; - xmlNode *cmd = NULL; - -+ conn = crm_ipc_new(target, 0); - if (!conn) { - return -ENOTCONN; - } -- - if (!crm_ipc_connect(conn)) { - crm_perror(LOG_ERR, "Connection to %s failed", target); - crm_ipc_destroy(conn); - return -ENOTCONN; - } - -- if(safe_str_eq(target, CRM_SYSTEM_CRMD)) { -- // The controller requires a hello message before sending a request -- rc = send_controller_hello(conn); -- if (rc < 0) { -- fprintf(stderr, "error: Could not register with controller: %s\n", -- pcmk_strerror(rc)); -- return rc; -- } -- } -- - crm_trace("Removing %s[%ld] from the %s membership cache", - node_name, nodeid, target); - -@@ -427,9 +396,9 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target) - crm_xml_add_int(cmd, PCMK__XA_ATTR_NODE_ID, (int) nodeid); - } - -- } else { -- cmd = create_request(CRM_OP_RM_NODE_CACHE, -- NULL, NULL, target, crm_system_name, pid_s); -+ } else { // Fencer or pacemakerd -+ cmd = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, target, -+ crm_system_name, NULL); - if (nodeid > 0) { - crm_xml_set_id(cmd, "%ld", nodeid); - } -@@ -441,6 +410,7 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target) - target, node_name, nodeid, rc); - - if (rc > 0) { -+ // @TODO Should this be done just once after all the rest? - rc = cib_remove_node(nodeid, node_name); - } - -@@ -455,12 +425,12 @@ tools_remove_node_cache(const char *node_name, long nodeid, const char *target) - static void - remove_node(const char *target_uname) - { -+ int rc; - int d = 0; - long nodeid = 0; - const char *node_name = NULL; - char *endptr = NULL; - const char *daemons[] = { -- CRM_SYSTEM_CRMD, - "stonith-ng", - T_ATTRD, - CRM_SYSTEM_MCP, -@@ -476,6 +446,12 @@ remove_node(const char *target_uname) - node_name = target_uname; - } - -+ rc = controller_remove_node(node_name, nodeid); -+ if (rc != pcmk_rc_ok) { -+ exit_code = pcmk_rc2exitc(rc); -+ return; -+ } -+ - for (d = 0; d < DIMOF(daemons); d++) { - if (tools_remove_node_cache(node_name, nodeid, daemons[d])) { - crm_err("Failed to connect to %s to remove node '%s'", -@@ -545,12 +521,34 @@ node_mcp_dispatch(const char *buffer, ssize_t length, gpointer userdata) - } - - static void -+lost_pacemakerd(gpointer user_data) -+{ -+ fprintf(stderr, "error: Lost connection to cluster\n"); -+ exit_code = CRM_EX_DISCONNECT; -+ g_main_loop_quit(mainloop); -+} -+ -+static void - run_pacemakerd_mainloop(void) - { - crm_ipc_t *ipc = NULL; - xmlNode *poke = NULL; -+ mainloop_io_t *source = NULL; - -- ipc = new_mainloop_for_ipc(CRM_SYSTEM_MCP, node_mcp_dispatch); -+ struct ipc_client_callbacks ipc_callbacks = { -+ .dispatch = node_mcp_dispatch, -+ .destroy = lost_pacemakerd -+ }; -+ -+ source = mainloop_add_ipc_client(CRM_SYSTEM_MCP, G_PRIORITY_DEFAULT, 0, -+ NULL, &ipc_callbacks); -+ ipc = mainloop_get_ipc_client(source); -+ if (ipc == NULL) { -+ fprintf(stderr, -+ "error: Could not connect to cluster (is it running?)\n"); -+ exit_code = CRM_EX_DISCONNECT; -+ return; -+ } - - // Sending anything will get us a list of nodes - poke = create_xml_node(NULL, "poke"); -@@ -558,6 +556,7 @@ run_pacemakerd_mainloop(void) - free_xml(poke); - - // Handle reply via node_mcp_dispatch() -+ mainloop = g_main_loop_new(NULL, FALSE); - g_main_loop_run(mainloop); - g_main_loop_unref(mainloop); - mainloop = NULL; --- -1.8.3.1 - - -From a361f764cb28630d440eec0f3e04a4f3812825eb Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 14 Apr 2020 16:05:15 -0500 -Subject: [PATCH 5/6] Refactor: tools: remove dead code from crm_node - ---- - tools/crm_node.c | 22 +--------------------- - 1 file changed, 1 insertion(+), 21 deletions(-) - -diff --git a/tools/crm_node.c b/tools/crm_node.c -index 1773a36..57c2ee5 100644 ---- a/tools/crm_node.c -+++ b/tools/crm_node.c -@@ -130,25 +130,6 @@ remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError * - return TRUE; - } - --/*! -- * \internal -- * \brief Exit crm_node -- * Clean up memory, and either quit mainloop (if running) or exit -- * -- * \param[in] value Exit status -- */ --static void --crm_node_exit(crm_exit_t value) --{ -- exit_code = value; -- -- if (mainloop && g_main_loop_is_running(mainloop)) { -- g_main_loop_quit(mainloop); -- } else { -- crm_exit(exit_code); -- } --} -- - static void - controller_event_cb(pcmk_ipc_api_t *controld_api, - enum pcmk_ipc_event event_type, crm_exit_t status, -@@ -660,6 +641,5 @@ done: - g_strfreev(processed_args); - g_clear_error(&error); - pcmk__free_arg_context(context); -- crm_node_exit(exit_code); -- return exit_code; -+ return crm_exit(exit_code); - } --- -1.8.3.1 - - -From 591944539259f6294450517770d95c7a02fc599c Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Mon, 20 Apr 2020 15:48:15 -0500 -Subject: [PATCH 6/6] Refactor: tools: convert crmadmin to use new controller - IPC model - ---- - tools/crmadmin.c | 484 ++++++++++++++++++++++++------------------------------- - 1 file changed, 208 insertions(+), 276 deletions(-) - -diff --git a/tools/crmadmin.c b/tools/crmadmin.c -index 3e9e959..4688458 100644 ---- a/tools/crmadmin.c -+++ b/tools/crmadmin.c -@@ -9,56 +9,44 @@ - - #include - --#include -- - #include --#include --#include -+#include -+#include // atoi() - --#include --#include --#include --#include - #include // gboolean, GMainLoop, etc. -+#include // xmlNode - - #include -+#include - #include - #include -- -+#include - #include - --#include -- - #define DEFAULT_MESSAGE_TIMEOUT_MS 30000 - - static guint message_timer_id = 0; - static guint message_timeout_ms = DEFAULT_MESSAGE_TIMEOUT_MS; - static GMainLoop *mainloop = NULL; --static crm_ipc_t *crmd_channel = NULL; --static char *admin_uuid = NULL; -- --gboolean do_init(void); --int do_work(void); --void crmadmin_ipc_connection_destroy(gpointer user_data); --int admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata); --int do_find_node_list(xmlNode * xml_node); -+ -+bool do_work(pcmk_ipc_api_t *api); -+void do_find_node_list(xmlNode *xml_node); - gboolean admin_message_timeout(gpointer data); - -+static enum { -+ cmd_none, -+ cmd_shutdown, -+ cmd_health, -+ cmd_elect_dc, -+ cmd_whois_dc, -+ cmd_list_nodes, -+} command = cmd_none; -+ - static gboolean BE_VERBOSE = FALSE; --static int expected_responses = 1; - static gboolean BASH_EXPORT = FALSE; --static gboolean DO_HEALTH = FALSE; --static gboolean DO_RESET = FALSE; --static gboolean DO_RESOURCE = FALSE; --static gboolean DO_ELECT_DC = FALSE; --static gboolean DO_WHOIS_DC = FALSE; --static gboolean DO_NODE_LIST = FALSE; - static gboolean BE_SILENT = FALSE; --static gboolean DO_RESOURCE_LIST = FALSE; --static const char *crmd_operation = NULL; - static char *dest_node = NULL; - static crm_exit_t exit_code = CRM_EX_OK; --static const char *sys_to = NULL; - - static pcmk__cli_option_t long_options[] = { - // long option, argument type, storage, short option, description, flags -@@ -133,7 +121,8 @@ static pcmk__cli_option_t long_options[] = { - }, - { - "bash-export", no_argument, NULL, 'B', -- "Create Bash export entries of the form 'export uname=uuid'\n", -+ "Display nodes as shell commands of the form 'export uname=uuid' " -+ "(valid with -N/--nodes)'\n", - pcmk__option_default - }, - { -@@ -142,13 +131,98 @@ static pcmk__cli_option_t long_options[] = { - }, - { - "-spacer-", no_argument, NULL, '-', -- " The -K and -E commands are rarely used and may be removed in " -- "future versions.", -+ "The -K and -E commands do not work and may be removed in a future " -+ "version.", - pcmk__option_default - }, - { 0, 0, 0, 0 } - }; - -+static void -+quit_main_loop(crm_exit_t ec) -+{ -+ exit_code = ec; -+ if (mainloop != NULL) { -+ GMainLoop *mloop = mainloop; -+ -+ mainloop = NULL; // Don't re-enter this block -+ pcmk_quit_main_loop(mloop, 10); -+ g_main_loop_unref(mloop); -+ } -+} -+ -+static void -+controller_event_cb(pcmk_ipc_api_t *controld_api, -+ enum pcmk_ipc_event event_type, crm_exit_t status, -+ void *event_data, void *user_data) -+{ -+ pcmk_controld_api_reply_t *reply = event_data; -+ -+ switch (event_type) { -+ case pcmk_ipc_event_disconnect: -+ if (exit_code == CRM_EX_DISCONNECT) { // Unexpected -+ fprintf(stderr, "error: Lost connection to controller\n"); -+ } -+ goto done; -+ break; -+ -+ case pcmk_ipc_event_reply: -+ break; -+ -+ default: -+ return; -+ } -+ -+ if (message_timer_id != 0) { -+ g_source_remove(message_timer_id); -+ message_timer_id = 0; -+ } -+ -+ if (status != CRM_EX_OK) { -+ fprintf(stderr, "error: Bad reply from controller: %s", -+ crm_exit_str(status)); -+ exit_code = status; -+ goto done; -+ } -+ -+ if (reply->reply_type != pcmk_controld_reply_ping) { -+ fprintf(stderr, "error: Unknown reply type %d from controller\n", -+ reply->reply_type); -+ goto done; -+ } -+ -+ // Parse desired information from reply -+ switch (command) { -+ case cmd_health: -+ printf("Status of %s@%s: %s (%s)\n", -+ reply->data.ping.sys_from, -+ reply->host_from, -+ reply->data.ping.fsa_state, -+ reply->data.ping.result); -+ if (BE_SILENT && (reply->data.ping.fsa_state != NULL)) { -+ fprintf(stderr, "%s\n", reply->data.ping.fsa_state); -+ } -+ exit_code = CRM_EX_OK; -+ break; -+ -+ case cmd_whois_dc: -+ printf("Designated Controller is: %s\n", reply->host_from); -+ if (BE_SILENT && (reply->host_from != NULL)) { -+ fprintf(stderr, "%s\n", reply->host_from); -+ } -+ exit_code = CRM_EX_OK; -+ break; -+ -+ default: // Not really possible here -+ exit_code = CRM_EX_SOFTWARE; -+ break; -+ } -+ -+done: -+ pcmk_disconnect_ipc(controld_api); -+ quit_main_loop(exit_code); -+} -+ - // \return Standard Pacemaker return code - static int - list_nodes() -@@ -181,6 +255,9 @@ main(int argc, char **argv) - int option_index = 0; - int argerr = 0; - int flag; -+ int rc; -+ pcmk_ipc_api_t *controld_api = NULL; -+ bool need_controld_api = true; - - crm_log_cli_init("crmadmin"); - pcmk__set_cli_options(NULL, " [options]", long_options, -@@ -211,33 +288,40 @@ main(int argc, char **argv) - pcmk__cli_help(flag, CRM_EX_OK); - break; - case 'D': -- DO_WHOIS_DC = TRUE; -+ command = cmd_whois_dc; - break; - case 'B': - BASH_EXPORT = TRUE; - break; - case 'K': -- DO_RESET = TRUE; -+ command = cmd_shutdown; - crm_trace("Option %c => %s", flag, optarg); -+ if (dest_node != NULL) { -+ free(dest_node); -+ } - dest_node = strdup(optarg); -- crmd_operation = CRM_OP_LOCAL_SHUTDOWN; - break; - case 'q': - BE_SILENT = TRUE; - break; - case 'S': -- DO_HEALTH = TRUE; -+ command = cmd_health; - crm_trace("Option %c => %s", flag, optarg); -+ if (dest_node != NULL) { -+ free(dest_node); -+ } - dest_node = strdup(optarg); - break; - case 'E': -- DO_ELECT_DC = TRUE; -+ command = cmd_elect_dc; - break; - case 'N': -- DO_NODE_LIST = TRUE; -+ command = cmd_list_nodes; -+ need_controld_api = false; - break; - case 'H': -- DO_HEALTH = TRUE; -+ fprintf(stderr, "Cluster-wide health option not supported\n"); -+ ++argerr; - break; - default: - printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag); -@@ -257,285 +341,133 @@ main(int argc, char **argv) - ++argerr; - } - -+ if (command == cmd_none) { -+ fprintf(stderr, "error: Must specify a command option\n\n"); -+ ++argerr; -+ } -+ - if (argerr) { - pcmk__cli_help('?', CRM_EX_USAGE); - } - -- if (do_init()) { -- int res = 0; -- -- res = do_work(); -- if (res > 0) { -- /* wait for the reply by creating a mainloop and running it until -- * the callbacks are invoked... -- */ -- mainloop = g_main_loop_new(NULL, FALSE); -- crm_trace("Waiting for %d replies from the local CRM", expected_responses); -- -- message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL); -- -- g_main_loop_run(mainloop); -- -- } else if (res < 0) { -- crm_err("No message to send"); -- exit_code = CRM_EX_ERROR; -+ // Connect to the controller if needed -+ if (need_controld_api) { -+ rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld); -+ if (controld_api == NULL) { -+ fprintf(stderr, "error: Could not connect to controller: %s\n", -+ pcmk_rc_str(rc)); -+ exit_code = pcmk_rc2exitc(rc); -+ goto done; - } -- } else { -- crm_warn("Init failed, could not perform requested operations"); -- exit_code = CRM_EX_UNAVAILABLE; -- } -- -- crm_trace("%s exiting normally", crm_system_name); -- return exit_code; --} -- --int --do_work(void) --{ -- int ret = 1; -- -- /* construct the request */ -- xmlNode *msg_data = NULL; -- gboolean all_is_good = TRUE; -- -- if (DO_HEALTH == TRUE) { -- crm_trace("Querying the system"); -- -- sys_to = CRM_SYSTEM_DC; -- -- if (dest_node != NULL) { -- sys_to = CRM_SYSTEM_CRMD; -- crmd_operation = CRM_OP_PING; -- -- if (BE_VERBOSE) { -- expected_responses = 1; -- } -- -- } else { -- crm_info("Cluster-wide health not available yet"); -- all_is_good = FALSE; -+ pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL); -+ rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main); -+ if (rc != pcmk_rc_ok) { -+ fprintf(stderr, "error: Could not connect to controller: %s\n", -+ pcmk_rc_str(rc)); -+ exit_code = pcmk_rc2exitc(rc); -+ goto done; - } -- -- } else if (DO_ELECT_DC) { -- /* tell the local node to initiate an election */ -- -- dest_node = NULL; -- sys_to = CRM_SYSTEM_CRMD; -- crmd_operation = CRM_OP_VOTE; -- ret = 0; /* no return message */ -- -- } else if (DO_WHOIS_DC) { -- dest_node = NULL; -- sys_to = CRM_SYSTEM_DC; -- crmd_operation = CRM_OP_PING; -- -- } else if (DO_NODE_LIST) { -- crm_exit(pcmk_rc2exitc(list_nodes())); -- -- } else if (DO_RESET) { -- /* tell dest_node to initiate the shutdown procedure -- * -- * if dest_node is NULL, the request will be sent to the -- * local node -- */ -- sys_to = CRM_SYSTEM_CRMD; -- ret = 0; /* no return message */ -- -- } else { -- crm_err("Unknown options"); -- all_is_good = FALSE; - } - -- if (all_is_good == FALSE) { -- crm_err("Creation of request failed. No message to send"); -- return -1; -+ if (do_work(controld_api)) { -+ // A reply is needed from controller, so run main loop to get it -+ exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects -+ mainloop = g_main_loop_new(NULL, FALSE); -+ message_timer_id = g_timeout_add(message_timeout_ms, -+ admin_message_timeout, NULL); -+ g_main_loop_run(mainloop); - } - --/* send it */ -- if (crmd_channel == NULL) { -- crm_err("The IPC connection is not valid, cannot send anything"); -- return -1; -- } -- -- if (sys_to == NULL) { -- if (dest_node != NULL) { -- sys_to = CRM_SYSTEM_CRMD; -- } else { -- sys_to = CRM_SYSTEM_DC; -- } -- } -- -- { -- xmlNode *cmd = create_request(crmd_operation, msg_data, dest_node, sys_to, -- crm_system_name, admin_uuid); -- -- crm_ipc_send(crmd_channel, cmd, 0, 0, NULL); -- free_xml(cmd); -- } -- -- return ret; --} -+done: -+ if (controld_api != NULL) { -+ pcmk_ipc_api_t *capi = controld_api; - --void --crmadmin_ipc_connection_destroy(gpointer user_data) --{ -- crm_err("Connection to controller was terminated"); -- if (mainloop) { -- g_main_loop_quit(mainloop); -- } else { -- crm_exit(CRM_EX_DISCONNECT); -+ controld_api = NULL; // Ensure we can't free this twice -+ pcmk_free_ipc_api(capi); - } --} -- --struct ipc_client_callbacks crm_callbacks = { -- .dispatch = admin_msg_callback, -- .destroy = crmadmin_ipc_connection_destroy --}; -- --gboolean --do_init(void) --{ -- mainloop_io_t *source = -- mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks); -- -- admin_uuid = pcmk__getpid_s(); -- -- crmd_channel = mainloop_get_ipc_client(source); -- -- if (DO_RESOURCE || DO_RESOURCE_LIST || DO_NODE_LIST) { -- return TRUE; -- -- } else if (crmd_channel != NULL) { -- xmlNode *xml = create_hello_message(admin_uuid, crm_system_name, "0", "1"); -- -- crm_ipc_send(crmd_channel, xml, 0, 0, NULL); -- return TRUE; -+ if (mainloop != NULL) { -+ g_main_loop_unref(mainloop); -+ mainloop = NULL; - } -- return FALSE; -+ return crm_exit(exit_code); - } - --static bool --validate_crm_message(xmlNode * msg, const char *sys, const char *uuid, const char *msg_type) -+// \return True if reply from controller is needed -+bool -+do_work(pcmk_ipc_api_t *controld_api) - { -- const char *type = NULL; -- const char *crm_msg_reference = NULL; -- -- if (msg == NULL) { -- return FALSE; -- } -+ bool need_reply = false; -+ int rc = pcmk_rc_ok; - -- type = crm_element_value(msg, F_CRM_MSG_TYPE); -- crm_msg_reference = crm_element_value(msg, XML_ATTR_REFERENCE); -- -- if (type == NULL) { -- crm_info("No message type defined."); -- return FALSE; -- -- } else if (msg_type != NULL && strcasecmp(msg_type, type) != 0) { -- crm_info("Expecting a (%s) message but received a (%s).", msg_type, type); -- return FALSE; -- } -- -- if (crm_msg_reference == NULL) { -- crm_info("No message crm_msg_reference defined."); -- return FALSE; -- } -- -- return TRUE; --} -- --int --admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata) --{ -- static int received_responses = 0; -- xmlNode *xml = string2xml(buffer); -- -- received_responses++; -- g_source_remove(message_timer_id); -- -- crm_log_xml_trace(xml, "ipc"); -- -- if (xml == NULL) { -- crm_info("XML in IPC message was not valid... " "discarding."); -- -- } else if (validate_crm_message(xml, crm_system_name, admin_uuid, XML_ATTR_RESPONSE) == FALSE) { -- crm_trace("Message was not a CRM response. Discarding."); -- -- } else if (DO_HEALTH) { -- xmlNode *data = get_message_xml(xml, F_CRM_DATA); -- const char *state = crm_element_value(data, XML_PING_ATTR_CRMDSTATE); -+ switch (command) { -+ case cmd_shutdown: -+ rc = pcmk_controld_api_shutdown(controld_api, dest_node); -+ break; - -- printf("Status of %s@%s: %s (%s)\n", -- crm_element_value(data, XML_PING_ATTR_SYSFROM), -- crm_element_value(xml, F_CRM_HOST_FROM), -- state, crm_element_value(data, XML_PING_ATTR_STATUS)); -+ case cmd_health: // dest_node != NULL -+ case cmd_whois_dc: // dest_node == NULL -+ rc = pcmk_controld_api_ping(controld_api, dest_node); -+ need_reply = true; -+ break; - -- if (BE_SILENT && state != NULL) { -- fprintf(stderr, "%s\n", state); -- } -+ case cmd_elect_dc: -+ rc = pcmk_controld_api_start_election(controld_api); -+ break; - -- } else if (DO_WHOIS_DC) { -- const char *dc = crm_element_value(xml, F_CRM_HOST_FROM); -+ case cmd_list_nodes: -+ rc = list_nodes(); -+ break; - -- printf("Designated Controller is: %s\n", dc); -- if (BE_SILENT && dc != NULL) { -- fprintf(stderr, "%s\n", dc); -- } -- crm_exit(CRM_EX_OK); -+ case cmd_none: // not actually possible here -+ break; - } -- -- free_xml(xml); -- -- if (received_responses >= expected_responses) { -- crm_trace("Received expected number (%d) of replies, exiting normally", -- expected_responses); -- crm_exit(CRM_EX_OK); -+ if (rc != pcmk_rc_ok) { -+ fprintf(stderr, "error: Command failed: %s", pcmk_rc_str(rc)); -+ exit_code = pcmk_rc2exitc(rc); - } -- -- message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL); -- return 0; -+ return need_reply; - } - - gboolean - admin_message_timeout(gpointer data) - { -- fprintf(stderr, "No messages received in %d seconds.. aborting\n", -- (int)message_timeout_ms / 1000); -- crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000); -- exit_code = CRM_EX_TIMEOUT; -- g_main_loop_quit(mainloop); -- return FALSE; -+ fprintf(stderr, -+ "error: No reply received from controller before timeout (%dms)\n", -+ message_timeout_ms); -+ message_timer_id = 0; -+ quit_main_loop(CRM_EX_TIMEOUT); -+ return FALSE; // Tells glib to remove source - } - --int -+void - do_find_node_list(xmlNode * xml_node) - { - int found = 0; - xmlNode *node = NULL; - xmlNode *nodes = get_object_root(XML_CIB_TAG_NODES, xml_node); - -- for (node = __xml_first_child_element(nodes); node != NULL; -- node = __xml_next_element(node)) { -+ for (node = first_named_child(nodes, XML_CIB_TAG_NODE); node != NULL; -+ node = crm_next_same_xml(node)) { - -- if (crm_str_eq((const char *)node->name, XML_CIB_TAG_NODE, TRUE)) { -+ if (BASH_EXPORT) { -+ printf("export %s=%s\n", -+ crm_element_value(node, XML_ATTR_UNAME), -+ crm_element_value(node, XML_ATTR_ID)); -+ } else { -+ const char *node_type = crm_element_value(node, XML_ATTR_TYPE); - -- if (BASH_EXPORT) { -- printf("export %s=%s\n", -- crm_element_value(node, XML_ATTR_UNAME), -- crm_element_value(node, XML_ATTR_ID)); -- } else { -- printf("%s node: %s (%s)\n", -- crm_element_value(node, XML_ATTR_TYPE), -- crm_element_value(node, XML_ATTR_UNAME), -- crm_element_value(node, XML_ATTR_ID)); -+ if (node_type == NULL) { -+ node_type = "member"; - } -- found++; -+ printf("%s node: %s (%s)\n", node_type, -+ crm_element_value(node, XML_ATTR_UNAME), -+ crm_element_value(node, XML_ATTR_ID)); - } -+ found++; - } -+ // @TODO List Pacemaker Remote nodes that don't have a entry - - if (found == 0) { -- printf("NO nodes configured\n"); -+ printf("No nodes configured\n"); - } -- -- return found; - } --- -1.8.3.1 - diff --git a/SOURCES/008-crm_node.patch b/SOURCES/008-crm_node.patch deleted file mode 100644 index 7d7c51a..0000000 --- a/SOURCES/008-crm_node.patch +++ /dev/null @@ -1,685 +0,0 @@ -From e01a1178fd8f5b99895683b3af9998e9485d12a4 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 24 Apr 2020 16:42:02 -0500 -Subject: [PATCH 1/3] Feature: controller: add new IPC API command for getting - node list - -This is based on and will replace the corresponding functionality in -pacemakerd. ---- - daemons/controld/controld_messages.c | 44 ++++++++++++++++++++++++++++-- - include/crm/common/ipc_controld.h | 13 +++++++++ - include/crm_internal.h | 1 + - lib/common/ipc_controld.c | 53 ++++++++++++++++++++++++++++++++++++ - 4 files changed, 109 insertions(+), 2 deletions(-) - -diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c -index 0be04d0..423f006 100644 ---- a/daemons/controld/controld_messages.c -+++ b/daemons/controld/controld_messages.c -@@ -375,10 +375,11 @@ relay_message(xmlNode * msg, gboolean originated_locally) - is_local = 0; - - } else if (is_for_crm) { -- if (safe_str_eq(task, CRM_OP_NODE_INFO)) { -+ if (safe_str_eq(task, CRM_OP_NODE_INFO) -+ || safe_str_eq(task, PCMK__CONTROLD_CMD_NODES)) { - /* Node info requests do not specify a host, which is normally - * treated as "all hosts", because the whole point is that the -- * client doesn't know the local node name. Always handle these -+ * client may not know the local node name. Always handle these - * requests locally. - */ - is_local = 1; -@@ -784,6 +785,42 @@ handle_ping(xmlNode *msg) - } - - /*! -+ * \brief Handle a PCMK__CONTROLD_CMD_NODES message -+ * -+ * \return Next FSA input -+ */ -+static enum crmd_fsa_input -+handle_node_list(xmlNode *request) -+{ -+ GHashTableIter iter; -+ crm_node_t *node = NULL; -+ xmlNode *reply = NULL; -+ xmlNode *reply_data = NULL; -+ -+ // Create message data for reply -+ reply_data = create_xml_node(NULL, XML_CIB_TAG_NODES); -+ g_hash_table_iter_init(&iter, crm_peer_cache); -+ while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) { -+ xmlNode *xml = create_xml_node(reply_data, XML_CIB_TAG_NODE); -+ -+ crm_xml_add_ll(xml, XML_ATTR_ID, (long long) node->id); // uint32_t -+ crm_xml_add(xml, XML_ATTR_UNAME, node->uname); -+ crm_xml_add(xml, XML_NODE_IN_CLUSTER, node->state); -+ } -+ -+ // Create and send reply -+ reply = create_reply(request, reply_data); -+ free_xml(reply_data); -+ if (reply) { -+ (void) relay_message(reply, TRUE); -+ free_xml(reply); -+ } -+ -+ // Nothing further to do -+ return I_NULL; -+} -+ -+/*! - * \brief Handle a CRM_OP_NODE_INFO request - * - * \param[in] msg Message XML -@@ -1080,6 +1117,9 @@ handle_request(xmlNode *stored_msg, enum crmd_fsa_cause cause) - - remote_ra_process_maintenance_nodes(xml); - -+ } else if (strcmp(op, PCMK__CONTROLD_CMD_NODES) == 0) { -+ return handle_node_list(stored_msg); -+ - /*========== (NOT_DC)-Only Actions ==========*/ - } else if (!AM_I_DC) { - -diff --git a/include/crm/common/ipc_controld.h b/include/crm/common/ipc_controld.h -index 0ebabfc..b817357 100644 ---- a/include/crm/common/ipc_controld.h -+++ b/include/crm/common/ipc_controld.h -@@ -22,6 +22,7 @@ extern "C" { - */ - - #include // bool -+#include // GList - #include // xmlNode - #include // pcmk_ipc_api_t - -@@ -32,8 +33,16 @@ enum pcmk_controld_api_reply { - pcmk_controld_reply_info, - pcmk_controld_reply_resource, - pcmk_controld_reply_ping, -+ pcmk_controld_reply_nodes, - }; - -+// Node information passed with pcmk_controld_reply_nodes -+typedef struct { -+ uint32_t id; -+ const char *uname; -+ const char *state; -+} pcmk_controld_api_node_t; -+ - /*! - * Controller reply passed to event callback - * -@@ -72,6 +81,9 @@ typedef struct { - const char *fsa_state; - const char *result; - } ping; -+ -+ // pcmk_controld_reply_nodes -+ GList *nodes; // list of pcmk_controld_api_node_t * - } data; - } pcmk_controld_api_reply_t; - -@@ -88,6 +100,7 @@ int pcmk_controld_api_refresh(pcmk_ipc_api_t *api, const char *target_node, - const char *provider, const char *type, - bool cib_only); - int pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name); -+int pcmk_controld_api_list_nodes(pcmk_ipc_api_t *api); - int pcmk_controld_api_shutdown(pcmk_ipc_api_t *api, const char *node_name); - int pcmk_controld_api_start_election(pcmk_ipc_api_t *api); - unsigned int pcmk_controld_api_replies_expected(pcmk_ipc_api_t *api); -diff --git a/include/crm_internal.h b/include/crm_internal.h -index fd56fc6..cf8999f 100644 ---- a/include/crm_internal.h -+++ b/include/crm_internal.h -@@ -122,6 +122,7 @@ pid_t pcmk_locate_sbd(void); - #define PCMK__ATTRD_CMD_SYNC_RESPONSE "sync-response" - #define PCMK__ATTRD_CMD_CLEAR_FAILURE "clear-failure" - -+#define PCMK__CONTROLD_CMD_NODES "list-nodes" - - /* - * Environment variables used by Pacemaker -diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c -index 22bb733..a733dd3 100644 ---- a/lib/common/ipc_controld.c -+++ b/lib/common/ipc_controld.c -@@ -120,6 +120,28 @@ set_ping_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data) - data->data.ping.result = crm_element_value(msg_data, XML_PING_ATTR_STATUS); - } - -+static void -+set_nodes_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data) -+{ -+ pcmk_controld_api_node_t *node_info; -+ -+ data->reply_type = pcmk_controld_reply_nodes; -+ for (xmlNode *node = first_named_child(msg_data, XML_CIB_TAG_NODE); -+ node != NULL; node = crm_next_same_xml(node)) { -+ -+ long long id_ll = 0; -+ -+ node_info = calloc(1, sizeof(pcmk_controld_api_node_t)); -+ crm_element_value_ll(node, XML_ATTR_ID, &id_ll); -+ if (id_ll > 0) { -+ node_info->id = id_ll; -+ } -+ node_info->uname = crm_element_value(node, XML_ATTR_UNAME); -+ node_info->state = crm_element_value(node, XML_NODE_IN_CLUSTER); -+ data->data.nodes = g_list_prepend(data->data.nodes, node_info); -+ } -+} -+ - static bool - reply_expected(pcmk_ipc_api_t *api, xmlNode *request) - { -@@ -201,6 +223,9 @@ dispatch(pcmk_ipc_api_t *api, xmlNode *reply) - } else if (!strcmp(value, CRM_OP_PING)) { - set_ping_data(&reply_data, msg_data); - -+ } else if (!strcmp(value, PCMK__CONTROLD_CMD_NODES)) { -+ set_nodes_data(&reply_data, msg_data); -+ - } else { - crm_debug("Unrecognizable controller message: unknown command '%s'", - value); -@@ -210,6 +235,11 @@ dispatch(pcmk_ipc_api_t *api, xmlNode *reply) - } - - pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data); -+ -+ // Free any reply data that was allocated -+ if (safe_str_eq(value, PCMK__CONTROLD_CMD_NODES)) { -+ g_list_free_full(reply_data.data.nodes, free); -+ } - } - - pcmk__ipc_methods_t * -@@ -376,6 +406,29 @@ pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name) - } - - /*! -+ * \brief Ask the controller for cluster information -+ * -+ * \param[in] api Controller connection -+ * -+ * \return Standard Pacemaker return code -+ * \note Event callback will get a reply of type pcmk_controld_reply_nodes. -+ */ -+int -+pcmk_controld_api_list_nodes(pcmk_ipc_api_t *api) -+{ -+ xmlNode *request; -+ int rc = EINVAL; -+ -+ request = create_controller_request(api, PCMK__CONTROLD_CMD_NODES, NULL, -+ NULL); -+ if (request != NULL) { -+ rc = send_controller_request(api, request, true); -+ free_xml(request); -+ } -+ return rc; -+} -+ -+/*! - * \internal - * \brief Ask the controller to shut down - * --- -1.8.3.1 - - -From 74e2d8d18bf534c1ec6f0e0f44a90772d393a553 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 2 Jul 2020 11:51:56 -0500 -Subject: [PATCH 2/3] Refactor: functionize numeric comparisons of strings - -This moves the guts of sort_node_uname() from libpe_status to a new function, -pcmk_numeric_strcasecmp(), in libcrmcommon, so it can be used with strings and -not just pe_node_t objects. ---- - include/crm/common/util.h | 1 + - lib/common/strings.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++ - lib/pengine/utils.c | 50 ++---------------------------------- - 3 files changed, 68 insertions(+), 48 deletions(-) - -diff --git a/include/crm/common/util.h b/include/crm/common/util.h -index 67d74d2..bb97b0a 100644 ---- a/include/crm/common/util.h -+++ b/include/crm/common/util.h -@@ -59,6 +59,7 @@ gboolean crm_strcase_equal(gconstpointer a, gconstpointer b); - char *crm_strdup_printf(char const *format, ...) __attribute__ ((__format__ (__printf__, 1, 2))); - int pcmk__parse_ll_range(const char *srcstring, long long *start, long long *end); - gboolean pcmk__str_in_list(GList *lst, const gchar *s); -+int pcmk_numeric_strcasecmp(const char *s1, const char *s2); - - # define safe_str_eq(a, b) crm_str_eq(a, b, FALSE) - # define crm_str_hash g_str_hash_traditional -diff --git a/lib/common/strings.c b/lib/common/strings.c -index 4562738..bd68ccf 100644 ---- a/lib/common/strings.c -+++ b/lib/common/strings.c -@@ -16,6 +16,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -715,3 +716,67 @@ pcmk__str_none_of(const char *s, ...) - - return g_list_find_custom(lst, s, (GCompareFunc) strcmp) != NULL; - } -+ -+/* -+ * \brief Sort strings, with numeric portions sorted numerically -+ * -+ * Sort two strings case-insensitively like strcasecmp(), but with any numeric -+ * portions of the string sorted numerically. This is particularly useful for -+ * node names (for example, "node10" will sort higher than "node9" but lower -+ * than "remotenode9"). -+ * -+ * \param[in] s1 First string to compare (must not be NULL) -+ * \param[in] s2 Second string to compare (must not be NULL) -+ * -+ * \retval -1 \p s1 comes before \p s2 -+ * \retval 0 \p s1 and \p s2 are equal -+ * \retval 1 \p s1 comes after \p s2 -+ */ -+int -+pcmk_numeric_strcasecmp(const char *s1, const char *s2) -+{ -+ while (*s1 && *s2) { -+ if (isdigit(*s1) && isdigit(*s2)) { -+ // If node names contain a number, sort numerically -+ -+ char *end1 = NULL; -+ char *end2 = NULL; -+ long num1 = strtol(s1, &end1, 10); -+ long num2 = strtol(s2, &end2, 10); -+ -+ // allow ordering e.g. 007 > 7 -+ size_t len1 = end1 - s1; -+ size_t len2 = end2 - s2; -+ -+ if (num1 < num2) { -+ return -1; -+ } else if (num1 > num2) { -+ return 1; -+ } else if (len1 < len2) { -+ return -1; -+ } else if (len1 > len2) { -+ return 1; -+ } -+ s1 = end1; -+ s2 = end2; -+ } else { -+ // Compare non-digits case-insensitively -+ int lower1 = tolower(*s1); -+ int lower2 = tolower(*s2); -+ -+ if (lower1 < lower2) { -+ return -1; -+ } else if (lower1 > lower2) { -+ return 1; -+ } -+ ++s1; -+ ++s2; -+ } -+ } -+ if (!*s1 && *s2) { -+ return -1; -+ } else if (*s1 && !*s2) { -+ return 1; -+ } -+ return 0; -+} -diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c -index ce3c260..584def4 100644 ---- a/lib/pengine/utils.c -+++ b/lib/pengine/utils.c -@@ -13,7 +13,6 @@ - #include - #include - --#include - #include - #include - -@@ -214,53 +213,8 @@ pe__node_list2table(GList *list) - gint - sort_node_uname(gconstpointer a, gconstpointer b) - { -- const char *name_a = ((const pe_node_t *) a)->details->uname; -- const char *name_b = ((const pe_node_t *) b)->details->uname; -- -- while (*name_a && *name_b) { -- if (isdigit(*name_a) && isdigit(*name_b)) { -- // If node names contain a number, sort numerically -- -- char *end_a = NULL; -- char *end_b = NULL; -- long num_a = strtol(name_a, &end_a, 10); -- long num_b = strtol(name_b, &end_b, 10); -- -- // allow ordering e.g. 007 > 7 -- size_t len_a = end_a - name_a; -- size_t len_b = end_b - name_b; -- -- if (num_a < num_b) { -- return -1; -- } else if (num_a > num_b) { -- return 1; -- } else if (len_a < len_b) { -- return -1; -- } else if (len_a > len_b) { -- return 1; -- } -- name_a = end_a; -- name_b = end_b; -- } else { -- // Compare non-digits case-insensitively -- int lower_a = tolower(*name_a); -- int lower_b = tolower(*name_b); -- -- if (lower_a < lower_b) { -- return -1; -- } else if (lower_a > lower_b) { -- return 1; -- } -- ++name_a; -- ++name_b; -- } -- } -- if (!*name_a && *name_b) { -- return -1; -- } else if (*name_a && !*name_b) { -- return 1; -- } -- return 0; -+ return pcmk_numeric_strcasecmp(((const pe_node_t *) a)->details->uname, -+ ((const pe_node_t *) b)->details->uname); - } - - /*! --- -1.8.3.1 - - -From 8461509158e06365122dc741c527c83c94e966ce Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 24 Apr 2020 19:35:19 -0500 -Subject: [PATCH 3/3] Fix: tools: crm_node -l and -p now work from Pacemaker - Remote nodes - -crm_node now asks the controller for the cluster node list, instead of -pacemakerd. This allows it to work from Pacemaker Remote nodes, since -controller IPC is proxied but pacemakerd IPC is not. ---- - tools/crm_node.c | 176 +++++++++++++++++++++---------------------------------- - 1 file changed, 67 insertions(+), 109 deletions(-) - -diff --git a/tools/crm_node.c b/tools/crm_node.c -index 57c2ee5..146454d 100644 ---- a/tools/crm_node.c -+++ b/tools/crm_node.c -@@ -130,6 +130,16 @@ remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError * - return TRUE; - } - -+static gint -+sort_node(gconstpointer a, gconstpointer b) -+{ -+ const pcmk_controld_api_node_t *node_a = a; -+ const pcmk_controld_api_node_t *node_b = b; -+ -+ return pcmk_numeric_strcasecmp((node_a->uname? node_a->uname : ""), -+ (node_b->uname? node_b->uname : "")); -+} -+ - static void - controller_event_cb(pcmk_ipc_api_t *controld_api, - enum pcmk_ipc_event event_type, crm_exit_t status, -@@ -157,15 +167,16 @@ controller_event_cb(pcmk_ipc_api_t *controld_api, - crm_exit_str(status)); - goto done; - } -- if (reply->reply_type != pcmk_controld_reply_info) { -- fprintf(stderr, "error: Unknown reply type %d from controller\n", -- reply->reply_type); -- goto done; -- } - - // Parse desired info from reply and display to user - switch (options.command) { - case 'i': -+ if (reply->reply_type != pcmk_controld_reply_info) { -+ fprintf(stderr, -+ "error: Unknown reply type %d from controller\n", -+ reply->reply_type); -+ goto done; -+ } - if (reply->data.node_info.id == 0) { - fprintf(stderr, - "error: Controller reply did not contain node ID\n"); -@@ -177,6 +188,12 @@ controller_event_cb(pcmk_ipc_api_t *controld_api, - - case 'n': - case 'N': -+ if (reply->reply_type != pcmk_controld_reply_info) { -+ fprintf(stderr, -+ "error: Unknown reply type %d from controller\n", -+ reply->reply_type); -+ goto done; -+ } - if (reply->data.node_info.uname == NULL) { - fprintf(stderr, "Node is not known to cluster\n"); - exit_code = CRM_EX_NOHOST; -@@ -186,6 +203,12 @@ controller_event_cb(pcmk_ipc_api_t *controld_api, - break; - - case 'q': -+ if (reply->reply_type != pcmk_controld_reply_info) { -+ fprintf(stderr, -+ "error: Unknown reply type %d from controller\n", -+ reply->reply_type); -+ goto done; -+ } - printf("%d\n", reply->data.node_info.have_quorum); - if (!(reply->data.node_info.have_quorum)) { - exit_code = CRM_EX_QUORUM; -@@ -193,6 +216,36 @@ controller_event_cb(pcmk_ipc_api_t *controld_api, - } - break; - -+ case 'l': -+ case 'p': -+ if (reply->reply_type != pcmk_controld_reply_nodes) { -+ fprintf(stderr, -+ "error: Unknown reply type %d from controller\n", -+ reply->reply_type); -+ goto done; -+ } -+ reply->data.nodes = g_list_sort(reply->data.nodes, sort_node); -+ for (GList *node_iter = reply->data.nodes; -+ node_iter != NULL; node_iter = node_iter->next) { -+ -+ pcmk_controld_api_node_t *node = node_iter->data; -+ const char *uname = (node->uname? node->uname : ""); -+ const char *state = (node->state? node->state : ""); -+ -+ if (options.command == 'l') { -+ printf("%lu %s %s\n", -+ (unsigned long) node->id, uname, state); -+ -+ // i.e. CRM_NODE_MEMBER, but we don't want to include cluster.h -+ } else if (!strcmp(state, "member")) { -+ printf("%s ", uname); -+ } -+ } -+ if (options.command == 'p') { -+ printf("\n"); -+ } -+ break; -+ - default: - fprintf(stderr, "internal error: Controller reply not expected\n"); - exit_code = CRM_EX_SOFTWARE; -@@ -207,7 +260,7 @@ done: - } - - static void --run_controller_mainloop(uint32_t nodeid) -+run_controller_mainloop(uint32_t nodeid, bool list_nodes) - { - pcmk_ipc_api_t *controld_api = NULL; - int rc; -@@ -233,7 +286,11 @@ run_controller_mainloop(uint32_t nodeid) - return; - } - -- rc = pcmk_controld_api_node_info(controld_api, nodeid); -+ if (list_nodes) { -+ rc = pcmk_controld_api_list_nodes(controld_api); -+ } else { -+ rc = pcmk_controld_api_node_info(controld_api, nodeid); -+ } - if (rc != pcmk_rc_ok) { - fprintf(stderr, "error: Could not ping controller: %s\n", - pcmk_rc_str(rc)); -@@ -263,7 +320,7 @@ print_node_name(void) - - } else { - // Otherwise ask the controller -- run_controller_mainloop(0); -+ run_controller_mainloop(0, false); - } - } - -@@ -444,105 +501,6 @@ remove_node(const char *target_uname) - exit_code = CRM_EX_OK; - } - --static gint --compare_node_xml(gconstpointer a, gconstpointer b) --{ -- const char *a_name = crm_element_value((xmlNode*) a, "uname"); -- const char *b_name = crm_element_value((xmlNode*) b, "uname"); -- -- return strcmp((a_name? a_name : ""), (b_name? b_name : "")); --} -- --static int --node_mcp_dispatch(const char *buffer, ssize_t length, gpointer userdata) --{ -- GList *nodes = NULL; -- xmlNode *node = NULL; -- xmlNode *msg = string2xml(buffer); -- const char *uname; -- const char *state; -- -- if (msg == NULL) { -- fprintf(stderr, "error: Could not understand pacemakerd response\n"); -- exit_code = CRM_EX_PROTOCOL; -- g_main_loop_quit(mainloop); -- return 0; -- } -- -- crm_log_xml_trace(msg, "message"); -- -- for (node = __xml_first_child(msg); node != NULL; node = __xml_next(node)) { -- nodes = g_list_insert_sorted(nodes, node, compare_node_xml); -- } -- -- for (GList *iter = nodes; iter; iter = iter->next) { -- node = (xmlNode*) iter->data; -- uname = crm_element_value(node, "uname"); -- state = crm_element_value(node, "state"); -- -- if (options.command == 'l') { -- int id = 0; -- -- crm_element_value_int(node, "id", &id); -- printf("%d %s %s\n", id, (uname? uname : ""), (state? state : "")); -- -- // This is CRM_NODE_MEMBER but we don't want to include cluster header -- } else if ((options.command == 'p') && safe_str_eq(state, "member")) { -- printf("%s ", (uname? uname : "")); -- } -- } -- if (options.command == 'p') { -- fprintf(stdout, "\n"); -- } -- -- free_xml(msg); -- exit_code = CRM_EX_OK; -- g_main_loop_quit(mainloop); -- return 0; --} -- --static void --lost_pacemakerd(gpointer user_data) --{ -- fprintf(stderr, "error: Lost connection to cluster\n"); -- exit_code = CRM_EX_DISCONNECT; -- g_main_loop_quit(mainloop); --} -- --static void --run_pacemakerd_mainloop(void) --{ -- crm_ipc_t *ipc = NULL; -- xmlNode *poke = NULL; -- mainloop_io_t *source = NULL; -- -- struct ipc_client_callbacks ipc_callbacks = { -- .dispatch = node_mcp_dispatch, -- .destroy = lost_pacemakerd -- }; -- -- source = mainloop_add_ipc_client(CRM_SYSTEM_MCP, G_PRIORITY_DEFAULT, 0, -- NULL, &ipc_callbacks); -- ipc = mainloop_get_ipc_client(source); -- if (ipc == NULL) { -- fprintf(stderr, -- "error: Could not connect to cluster (is it running?)\n"); -- exit_code = CRM_EX_DISCONNECT; -- return; -- } -- -- // Sending anything will get us a list of nodes -- poke = create_xml_node(NULL, "poke"); -- crm_ipc_send(ipc, poke, 0, 0, NULL); -- free_xml(poke); -- -- // Handle reply via node_mcp_dispatch() -- mainloop = g_main_loop_new(NULL, FALSE); -- g_main_loop_run(mainloop); -- g_main_loop_unref(mainloop); -- mainloop = NULL; --} -- - static GOptionContext * - build_arg_context(pcmk__common_args_t *args, GOptionGroup *group) { - GOptionContext *context = NULL; -@@ -627,11 +585,11 @@ main(int argc, char **argv) - case 'i': - case 'q': - case 'N': -- run_controller_mainloop(options.nodeid); -+ run_controller_mainloop(options.nodeid, false); - break; - case 'l': - case 'p': -- run_pacemakerd_mainloop(); -+ run_controller_mainloop(0, true); - break; - default: - break; --- -1.8.3.1 - diff --git a/SOURCES/009-timeout-log.patch b/SOURCES/009-timeout-log.patch deleted file mode 100644 index 74447bc..0000000 --- a/SOURCES/009-timeout-log.patch +++ /dev/null @@ -1,34 +0,0 @@ -From b542a8f667002519fbc07693a796553746c43c12 Mon Sep 17 00:00:00 2001 -From: Reid Wahl -Date: Sat, 11 Jul 2020 18:31:55 -0700 -Subject: [PATCH] Log: controld: Show action timer plus cluster-delay in - action_timer cb - -`action_timer_callback()` prints a misleading error message. If it -times out waiting for an action result, the error message prints the -timeout value followed by "(action timeout plus cluster-delay)". -However, only the `action->timeout` value is displayed. `cluster-delay` -is not added in. - -Resolves: RHBZ#1856035 ---- - daemons/controld/controld_te_callbacks.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/daemons/controld/controld_te_callbacks.c b/daemons/controld/controld_te_callbacks.c -index 8506f26..6ddaffe 100644 ---- a/daemons/controld/controld_te_callbacks.c -+++ b/daemons/controld/controld_te_callbacks.c -@@ -697,7 +697,8 @@ action_timer_callback(gpointer data) - crm_err("Node %s did not send %s result (via %s) within %dms " - "(action timeout plus cluster-delay)", - (on_node? on_node : ""), (task? task : "unknown action"), -- (via_node? via_node : "controller"), timer->timeout); -+ (via_node? via_node : "controller"), -+ timer->timeout + transition_graph->network_delay); - print_action(LOG_ERR, "Aborting transition, action lost: ", timer->action); - - timer->action->failed = TRUE; --- -1.8.3.1 - diff --git a/SOURCES/010-crm_mon.patch b/SOURCES/010-crm_mon.patch deleted file mode 100644 index 414b68a..0000000 --- a/SOURCES/010-crm_mon.patch +++ /dev/null @@ -1,476 +0,0 @@ -From 7056ae08bfa5cafeec9c454cb40aefa7553af6df Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Thu, 16 Jul 2020 12:53:24 -0400 -Subject: [PATCH 1/4] Fix: libcrmcommon: Set out->priv to NULL in free_priv. - -init won't do anything if priv is not NULL, so when the private data is -freed, also set it to NULL. This prevents segfaults when reset is -called. ---- - lib/common/output_html.c | 1 + - lib/common/output_log.c | 1 + - lib/common/output_text.c | 1 + - lib/common/output_xml.c | 1 + - tools/crm_mon_curses.c | 1 + - 5 files changed, 5 insertions(+) - -diff --git a/lib/common/output_html.c b/lib/common/output_html.c -index c8f0088..fc06641 100644 ---- a/lib/common/output_html.c -+++ b/lib/common/output_html.c -@@ -72,6 +72,7 @@ html_free_priv(pcmk__output_t *out) { - g_queue_free(priv->parent_q); - g_slist_free(priv->errors); - free(priv); -+ out->priv = NULL; - } - - static bool -diff --git a/lib/common/output_log.c b/lib/common/output_log.c -index 5b45ce4..0208046 100644 ---- a/lib/common/output_log.c -+++ b/lib/common/output_log.c -@@ -44,6 +44,7 @@ log_free_priv(pcmk__output_t *out) { - - g_queue_free(priv->prefixes); - free(priv); -+ out->priv = NULL; - } - - static bool -diff --git a/lib/common/output_text.c b/lib/common/output_text.c -index 54c409a..8f15849 100644 ---- a/lib/common/output_text.c -+++ b/lib/common/output_text.c -@@ -43,6 +43,7 @@ text_free_priv(pcmk__output_t *out) { - - g_queue_free(priv->parent_q); - free(priv); -+ out->priv = NULL; - } - - static bool -diff --git a/lib/common/output_xml.c b/lib/common/output_xml.c -index 8565bfe..858da3f 100644 ---- a/lib/common/output_xml.c -+++ b/lib/common/output_xml.c -@@ -54,6 +54,7 @@ xml_free_priv(pcmk__output_t *out) { - g_queue_free(priv->parent_q); - g_slist_free(priv->errors); - free(priv); -+ out->priv = NULL; - } - - static bool -diff --git a/tools/crm_mon_curses.c b/tools/crm_mon_curses.c -index d93b847..e9cc023 100644 ---- a/tools/crm_mon_curses.c -+++ b/tools/crm_mon_curses.c -@@ -46,6 +46,7 @@ curses_free_priv(pcmk__output_t *out) { - - g_queue_free(priv->parent_q); - free(priv); -+ out->priv = NULL; - } - - static bool --- -1.8.3.1 - - -From 3779152993ca0e88dc407c918882568217f1b630 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Thu, 16 Jul 2020 13:50:24 -0400 -Subject: [PATCH 2/4] Fix: libcrmcommon: Make reset and finish work more - similarly. - -When finish is called for HTML and XML output formats, various extra -nodes and headers are added, errors are added, etc. None of this stuff -happens on reset. For the HTML format, this also means things like the -CGI headers and title don't get added when reset is called. Make these -two functions much more similar. - -Regression in 2.0.3. - -See: rhbz#1857728 ---- - lib/common/output_html.c | 26 ++++++++++++++++---------- - lib/common/output_xml.c | 30 ++++++++++++++++-------------- - 2 files changed, 32 insertions(+), 24 deletions(-) - -diff --git a/lib/common/output_html.c b/lib/common/output_html.c -index fc06641..6127df2 100644 ---- a/lib/common/output_html.c -+++ b/lib/common/output_html.c -@@ -113,18 +113,11 @@ add_error_node(gpointer data, gpointer user_data) { - } - - static void --html_finish(pcmk__output_t *out, crm_exit_t exit_status, bool print, void **copy_dest) { -+finish_reset_common(pcmk__output_t *out, crm_exit_t exit_status, bool print) { - private_data_t *priv = out->priv; - htmlNodePtr head_node = NULL; - htmlNodePtr charset_node = NULL; - -- /* If root is NULL, html_init failed and we are being called from pcmk__output_free -- * in the pcmk__output_new path. -- */ -- if (priv == NULL || priv->root == NULL) { -- return; -- } -- - if (cgi_output && print) { - fprintf(out->dest, "Content-Type: text/html\n\n"); - } -@@ -174,6 +167,20 @@ html_finish(pcmk__output_t *out, crm_exit_t exit_status, bool print, void **copy - if (print) { - htmlDocDump(out->dest, priv->root->doc); - } -+} -+ -+static void -+html_finish(pcmk__output_t *out, crm_exit_t exit_status, bool print, void **copy_dest) { -+ private_data_t *priv = out->priv; -+ -+ /* If root is NULL, html_init failed and we are being called from pcmk__output_free -+ * in the pcmk__output_new path. -+ */ -+ if (priv == NULL || priv->root == NULL) { -+ return; -+ } -+ -+ finish_reset_common(out, exit_status, print); - - if (copy_dest != NULL) { - *copy_dest = copy_xml(priv->root); -@@ -185,8 +192,7 @@ html_reset(pcmk__output_t *out) { - CRM_ASSERT(out != NULL); - - if (out->priv != NULL) { -- private_data_t *priv = out->priv; -- htmlDocDump(out->dest, priv->root->doc); -+ finish_reset_common(out, CRM_EX_OK, true); - } - - html_free_priv(out); -diff --git a/lib/common/output_xml.c b/lib/common/output_xml.c -index 858da3f..b64a71d 100644 ---- a/lib/common/output_xml.c -+++ b/lib/common/output_xml.c -@@ -106,17 +106,10 @@ add_error_node(gpointer data, gpointer user_data) { - } - - static void --xml_finish(pcmk__output_t *out, crm_exit_t exit_status, bool print, void **copy_dest) { -+finish_reset_common(pcmk__output_t *out, crm_exit_t exit_status, bool print) { - xmlNodePtr node; - private_data_t *priv = out->priv; - -- /* If root is NULL, xml_init failed and we are being called from pcmk__output_free -- * in the pcmk__output_new path. -- */ -- if (priv == NULL || priv->root == NULL) { -- return; -- } -- - if (legacy_xml) { - GSList *node = priv->errors; - -@@ -148,6 +141,20 @@ xml_finish(pcmk__output_t *out, crm_exit_t exit_status, bool print, void **copy_ - fprintf(out->dest, "%s", buf); - free(buf); - } -+} -+ -+static void -+xml_finish(pcmk__output_t *out, crm_exit_t exit_status, bool print, void **copy_dest) { -+ private_data_t *priv = out->priv; -+ -+ /* If root is NULL, xml_init failed and we are being called from pcmk__output_free -+ * in the pcmk__output_new path. -+ */ -+ if (priv == NULL || priv->root == NULL) { -+ return; -+ } -+ -+ finish_reset_common(out, exit_status, print); - - if (copy_dest != NULL) { - *copy_dest = copy_xml(priv->root); -@@ -156,15 +163,10 @@ xml_finish(pcmk__output_t *out, crm_exit_t exit_status, bool print, void **copy_ - - static void - xml_reset(pcmk__output_t *out) { -- char *buf = NULL; -- - CRM_ASSERT(out != NULL); - - if (out->priv != NULL) { -- private_data_t *priv = out->priv; -- buf = dump_xml_formatted_with_text(priv->root); -- fprintf(out->dest, "%s", buf); -- free(buf); -+ finish_reset_common(out, CRM_EX_OK, true); - } - - xml_free_priv(out); --- -1.8.3.1 - - -From 0f8e4ca5d9a429c934f1e91a1bdf572efd07e0db Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Thu, 16 Jul 2020 16:09:08 -0400 -Subject: [PATCH 3/4] Fix: tools, libcrmcommon: Reopen the output dest on - reset. - -This is needed when running crm_mon as a daemon. When we do a reset, -we need to clear out any existing output destination and start writing -again from the beginning. This really only matters when the destination -is a file. - -The extra freopen at the end of crm_mon is to handle when crm_mon is -killed. We need to reset the output file to its beginning before -calling finish. ---- - lib/common/output_html.c | 3 +++ - lib/common/output_log.c | 3 +++ - lib/common/output_text.c | 3 +++ - lib/common/output_xml.c | 3 +++ - tools/crm_mon.c | 9 +++++++++ - 5 files changed, 21 insertions(+) - -diff --git a/lib/common/output_html.c b/lib/common/output_html.c -index 6127df2..6e21031 100644 ---- a/lib/common/output_html.c -+++ b/lib/common/output_html.c -@@ -191,6 +191,9 @@ static void - html_reset(pcmk__output_t *out) { - CRM_ASSERT(out != NULL); - -+ out->dest = freopen(NULL, "w", out->dest); -+ CRM_ASSERT(out->dest != NULL); -+ - if (out->priv != NULL) { - finish_reset_common(out, CRM_EX_OK, true); - } -diff --git a/lib/common/output_log.c b/lib/common/output_log.c -index 0208046..8422ac2 100644 ---- a/lib/common/output_log.c -+++ b/lib/common/output_log.c -@@ -72,6 +72,9 @@ static void - log_reset(pcmk__output_t *out) { - CRM_ASSERT(out != NULL); - -+ out->dest = freopen(NULL, "w", out->dest); -+ CRM_ASSERT(out->dest != NULL); -+ - log_free_priv(out); - log_init(out); - } -diff --git a/lib/common/output_text.c b/lib/common/output_text.c -index 8f15849..2f7e5b0 100644 ---- a/lib/common/output_text.c -+++ b/lib/common/output_text.c -@@ -75,6 +75,9 @@ static void - text_reset(pcmk__output_t *out) { - CRM_ASSERT(out != NULL); - -+ out->dest = freopen(NULL, "w", out->dest); -+ CRM_ASSERT(out->dest != NULL); -+ - text_free_priv(out); - text_init(out); - } -diff --git a/lib/common/output_xml.c b/lib/common/output_xml.c -index b64a71d..9f8e01b 100644 ---- a/lib/common/output_xml.c -+++ b/lib/common/output_xml.c -@@ -165,6 +165,9 @@ static void - xml_reset(pcmk__output_t *out) { - CRM_ASSERT(out != NULL); - -+ out->dest = freopen(NULL, "w", out->dest); -+ CRM_ASSERT(out->dest != NULL); -+ - if (out->priv != NULL) { - finish_reset_common(out, CRM_EX_OK, true); - } -diff --git a/tools/crm_mon.c b/tools/crm_mon.c -index b2e143b..10624c1 100644 ---- a/tools/crm_mon.c -+++ b/tools/crm_mon.c -@@ -2014,6 +2014,10 @@ mon_refresh_display(gpointer user_data) - break; - } - -+ if (options.daemonize) { -+ out->reset(out); -+ } -+ - stonith_history_free(stonith_history); - stonith_history = NULL; - pe_reset_working_set(mon_data_set); -@@ -2179,6 +2183,11 @@ clean_up(crm_exit_t exit_code) - * crm_mon to be able to do so. - */ - if (out != NULL) { -+ if (options.daemonize) { -+ out->dest = freopen(NULL, "w", out->dest); -+ CRM_ASSERT(out->dest != NULL); -+ } -+ - switch (output_format) { - case mon_output_cgi: - case mon_output_html: --- -1.8.3.1 - - -From b655c039414d2c7af77c3532222b04684ef1f3d0 Mon Sep 17 00:00:00 2001 -From: Chris Lumens -Date: Fri, 17 Jul 2020 10:58:32 -0400 -Subject: [PATCH 4/4] Fix: tools: Add the http-equiv header to crm_mon at the - right time. - -This header is only getting added on termination, which is not a lot of -help if you're running crm_mon in daemonize mode. Putting this header -in at the right time requires a couple changes: - -* pcmk__html_add_header doesn't need a parent argument. It was not -being used in the first place. - -* The extra_headers list in output_html.c should not be freed in the -reset function. This means it would get freed after every time the -daemonized output is refreshed, which means the header would have to be -added every time too. extra_headers will now only be freed when the -program exits. This is a behavior change, but I can't see why it's -going to be a problem. - -* To support that, we need to copy each item in the extra_headers list -when it gets added to the output XML document. This prevents segfaults -when we later free that document. - -* handle_html_output no longer needs to exist. That function only -existed to add the http-equiv header at the end, which is wrong. ---- - include/crm/common/output.h | 5 ++--- - lib/common/output_html.c | 7 ++++--- - tools/crm_mon.c | 26 +++++++------------------- - 3 files changed, 13 insertions(+), 25 deletions(-) - -diff --git a/include/crm/common/output.h b/include/crm/common/output.h -index e7c9417..186bcfe 100644 ---- a/include/crm/common/output.h -+++ b/include/crm/common/output.h -@@ -703,15 +703,14 @@ pcmk__output_create_html_node(pcmk__output_t *out, const char *element_name, con - * the following code would generate the tag "": - * - * \code -- * pcmk__html_add_header(parent, "meta", "http-equiv", "refresh", "content", "19", NULL); -+ * pcmk__html_add_header("meta", "http-equiv", "refresh", "content", "19", NULL); - * \endcode - * -- * \param[in,out] parent The node that will be the parent of the new node. - * \param[in] name The HTML tag for the new node. - * \param[in] ... A NULL-terminated key/value list of attributes. - */ - void --pcmk__html_add_header(xmlNodePtr parent, const char *name, ...) -+pcmk__html_add_header(const char *name, ...) - G_GNUC_NULL_TERMINATED; - - #ifdef __cplusplus -diff --git a/lib/common/output_html.c b/lib/common/output_html.c -index 6e21031..259e412 100644 ---- a/lib/common/output_html.c -+++ b/lib/common/output_html.c -@@ -139,7 +139,7 @@ finish_reset_common(pcmk__output_t *out, crm_exit_t exit_status, bool print) { - - /* Add any extra header nodes the caller might have created. */ - for (int i = 0; i < g_slist_length(extra_headers); i++) { -- xmlAddChild(head_node, g_slist_nth_data(extra_headers, i)); -+ xmlAddChild(head_node, xmlCopyNode(g_slist_nth_data(extra_headers, i), 1)); - } - - /* Stylesheets are included two different ways. The first is via a built-in -@@ -185,6 +185,8 @@ html_finish(pcmk__output_t *out, crm_exit_t exit_status, bool print, void **copy - if (copy_dest != NULL) { - *copy_dest = copy_xml(priv->root); - } -+ -+ g_slist_free_full(extra_headers, (GDestroyNotify) xmlFreeNode); - } - - static void -@@ -199,7 +201,6 @@ html_reset(pcmk__output_t *out) { - } - - html_free_priv(out); -- g_slist_free_full(extra_headers, (GDestroyNotify) xmlFreeNode); - html_init(out); - } - -@@ -412,7 +413,7 @@ pcmk__output_create_html_node(pcmk__output_t *out, const char *element_name, con - } - - void --pcmk__html_add_header(xmlNodePtr parent, const char *name, ...) { -+pcmk__html_add_header(const char *name, ...) { - htmlNodePtr header_node; - va_list ap; - -diff --git a/tools/crm_mon.c b/tools/crm_mon.c -index 10624c1..7fd2b9c 100644 ---- a/tools/crm_mon.c -+++ b/tools/crm_mon.c -@@ -1346,6 +1346,12 @@ main(int argc, char **argv) - options.mon_ops |= mon_op_print_timing | mon_op_inactive_resources; - } - -+ if ((output_format == mon_output_html || output_format == mon_output_cgi) && -+ out->dest != stdout) { -+ pcmk__html_add_header("meta", "http-equiv", "refresh", "content", -+ crm_itoa(options.reconnect_msec/1000), NULL); -+ } -+ - crm_info("Starting %s", crm_system_name); - - if (cib) { -@@ -2106,15 +2112,6 @@ clean_up_connections(void) - } - } - --static void --handle_html_output(crm_exit_t exit_code) { -- xmlNodePtr html = NULL; -- -- pcmk__html_add_header(html, "meta", "http-equiv", "refresh", "content", -- crm_itoa(options.reconnect_msec/1000), NULL); -- out->finish(out, exit_code, true, (void **) &html); --} -- - /* - * De-init ncurses, disconnect from the CIB manager, disconnect fencing, - * deallocate memory and show usage-message if requested. -@@ -2188,16 +2185,7 @@ clean_up(crm_exit_t exit_code) - CRM_ASSERT(out->dest != NULL); - } - -- switch (output_format) { -- case mon_output_cgi: -- case mon_output_html: -- handle_html_output(exit_code); -- break; -- -- default: -- out->finish(out, exit_code, true, NULL); -- break; -- } -+ out->finish(out, exit_code, true, NULL); - - pcmk__output_free(out); - pcmk__unregister_formats(); --- -1.8.3.1 - diff --git a/SOURCES/011-cts.patch b/SOURCES/011-cts.patch deleted file mode 100644 index 1b0ec7f..0000000 --- a/SOURCES/011-cts.patch +++ /dev/null @@ -1,270 +0,0 @@ -From 4e190ebc5460563bae2586b28afb0415f2eb3d1a Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Wed, 1 Jul 2020 20:38:16 -0500 -Subject: [PATCH 1/4] Test: CTS: libqb shared memory creates directories now - -... so use "rm -rf" instead of "rm -f" ---- - cts/CTS.py.in | 2 +- - cts/CTSaudits.py | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/cts/CTS.py.in b/cts/CTS.py.in -index c418318..091bb1f 100644 ---- a/cts/CTS.py.in -+++ b/cts/CTS.py.in -@@ -546,7 +546,7 @@ class ClusterManager(UserDict): - if self.rsh(node, self.templates["StopCmd"]) == 0: - # Make sure we can continue even if corosync leaks - # fdata-* is the old name -- #self.rsh(node, "rm -f /dev/shm/qb-* /dev/shm/fdata-*") -+ #self.rsh(node, "rm -rf /dev/shm/qb-* /dev/shm/fdata-*") - self.ShouldBeStatus[node] = "down" - self.cluster_stable(self.Env["DeadTime"]) - return 1 -diff --git a/cts/CTSaudits.py b/cts/CTSaudits.py -index b7e0827..cc82171 100755 ---- a/cts/CTSaudits.py -+++ b/cts/CTSaudits.py -@@ -233,7 +233,7 @@ class FileAudit(ClusterAudit): - for line in lsout: - self.CM.debug("ps[%s]: %s" % (node, line)) - -- self.CM.rsh(node, "rm -f /dev/shm/qb-*") -+ self.CM.rsh(node, "rm -rf /dev/shm/qb-*") - - else: - self.CM.debug("Skipping %s" % node) --- -1.8.3.1 - - -From 4316507d50d51c7864d8d34aac1da31a232b9f42 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 2 Jul 2020 16:09:20 -0500 -Subject: [PATCH 2/4] Test: CTS: ignore error logged by recent pcs versions - -... because it is expected when a node is fenced, and we should already see -pacemaker errors if a node is unexpectedly fenced ---- - cts/patterns.py | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/cts/patterns.py b/cts/patterns.py -index 96d6471..7eed90c 100644 ---- a/cts/patterns.py -+++ b/cts/patterns.py -@@ -21,6 +21,10 @@ class BasePatterns(object): - - # Logging bug in some versions of libvirtd - r"libvirtd.*: internal error: Failed to parse PCI config address", -+ -+ # pcs can log this when node is fenced, but fencing is OK in some -+ # tests (and we will catch it in pacemaker logs when not OK) -+ r"pcs.daemon:No response from: .* request: get_configs, error:", - ] - self.BadNews = [] - self.components = {} --- -1.8.3.1 - - -From 598ae0f65bad6ed16978d1ab6e24e8e358e0a1a4 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 2 Jul 2020 20:40:00 -0500 -Subject: [PATCH 3/4] Low: libcrmcommon: avoid assertion on controller protocol - errors - -Previously, after a protocol error, we would set reply to NULL and then try to -call crm_element_value() on it, which would log an assertion. ---- - lib/common/ipc_controld.c | 46 ++++++++++++++++++++++------------------------ - 1 file changed, 22 insertions(+), 24 deletions(-) - -diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c -index 5917cc5..22cb9e0 100644 ---- a/lib/common/ipc_controld.c -+++ b/lib/common/ipc_controld.c -@@ -187,53 +187,51 @@ dispatch(pcmk_ipc_api_t *api, xmlNode *reply) - crm_debug("Unrecognizable controller message: invalid message type '%s'", - crm_str(value)); - status = CRM_EX_PROTOCOL; -- reply = NULL; -+ goto done; - } - - if (crm_element_value(reply, XML_ATTR_REFERENCE) == NULL) { - crm_debug("Unrecognizable controller message: no reference"); - status = CRM_EX_PROTOCOL; -- reply = NULL; -+ goto done; - } - - value = crm_element_value(reply, F_CRM_TASK); - if (value == NULL) { - crm_debug("Unrecognizable controller message: no command name"); - status = CRM_EX_PROTOCOL; -- reply = NULL; -+ goto done; - } - - // Parse useful info from reply - -- if (reply != NULL) { -- reply_data.feature_set = crm_element_value(reply, XML_ATTR_VERSION); -- reply_data.host_from = crm_element_value(reply, F_CRM_HOST_FROM); -- msg_data = get_message_xml(reply, F_CRM_DATA); -+ reply_data.feature_set = crm_element_value(reply, XML_ATTR_VERSION); -+ reply_data.host_from = crm_element_value(reply, F_CRM_HOST_FROM); -+ msg_data = get_message_xml(reply, F_CRM_DATA); - -- if (!strcmp(value, CRM_OP_REPROBE)) { -- reply_data.reply_type = pcmk_controld_reply_reprobe; -+ if (!strcmp(value, CRM_OP_REPROBE)) { -+ reply_data.reply_type = pcmk_controld_reply_reprobe; - -- } else if (!strcmp(value, CRM_OP_NODE_INFO)) { -- set_node_info_data(&reply_data, msg_data); -+ } else if (!strcmp(value, CRM_OP_NODE_INFO)) { -+ set_node_info_data(&reply_data, msg_data); - -- } else if (!strcmp(value, CRM_OP_INVOKE_LRM)) { -- reply_data.reply_type = pcmk_controld_reply_resource; -- reply_data.data.resource.node_state = msg_data; -+ } else if (!strcmp(value, CRM_OP_INVOKE_LRM)) { -+ reply_data.reply_type = pcmk_controld_reply_resource; -+ reply_data.data.resource.node_state = msg_data; - -- } else if (!strcmp(value, CRM_OP_PING)) { -- set_ping_data(&reply_data, msg_data); -+ } else if (!strcmp(value, CRM_OP_PING)) { -+ set_ping_data(&reply_data, msg_data); - -- } else if (!strcmp(value, PCMK__CONTROLD_CMD_NODES)) { -- set_nodes_data(&reply_data, msg_data); -+ } else if (!strcmp(value, PCMK__CONTROLD_CMD_NODES)) { -+ set_nodes_data(&reply_data, msg_data); - -- } else { -- crm_debug("Unrecognizable controller message: unknown command '%s'", -- value); -- status = CRM_EX_PROTOCOL; -- reply = NULL; -- } -+ } else { -+ crm_debug("Unrecognizable controller message: unknown command '%s'", -+ value); -+ status = CRM_EX_PROTOCOL; - } - -+done: - pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data); - - // Free any reply data that was allocated --- -1.8.3.1 - - -From 5ae4101b60f8c0cd96eb2097a65a59aaa1750d73 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Fri, 17 Jul 2020 17:20:23 -0500 -Subject: [PATCH 4/4] Log: fencer: don't log assertion if unable to create full - request reply - -Previously, we would log an assertion and a warning if asked to create a reply -to a NULL request. However there is a possible sequence for this to happen: - -- Some nodes are up and some down at cluster start-up -- One node is elected DC and schedules fencing of the down nodes -- Fencing is initiated for one of the down nodes -- One of the other down nodes comes up and is elected DC -- The fencing result comes back and all peers (including new DC) are notified -- New DC tries to create a notification for its client (the controller) - but doesn't know anything about the initial request - -For now, just log a warning and drop the assertion. Longer term, maybe we -should synchronize in-flight request information when a fencer joins the -process group. ---- - daemons/fenced/fenced_commands.c | 55 +++++++++++++++++++++++----------------- - 1 file changed, 32 insertions(+), 23 deletions(-) - -diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c -index 05c5437..9c27d61 100644 ---- a/daemons/fenced/fenced_commands.c -+++ b/daemons/fenced/fenced_commands.c -@@ -2336,22 +2336,8 @@ stonith_fence(xmlNode * msg) - xmlNode * - stonith_construct_reply(xmlNode * request, const char *output, xmlNode * data, int rc) - { -- int lpc = 0; - xmlNode *reply = NULL; - -- const char *name = NULL; -- const char *value = NULL; -- -- const char *names[] = { -- F_STONITH_OPERATION, -- F_STONITH_CALLID, -- F_STONITH_CLIENTID, -- F_STONITH_CLIENTNAME, -- F_STONITH_REMOTE_OP_ID, -- F_STONITH_CALLOPTS -- }; -- -- crm_trace("Creating a basic reply"); - reply = create_xml_node(NULL, T_STONITH_REPLY); - - crm_xml_add(reply, "st_origin", __FUNCTION__); -@@ -2359,16 +2345,39 @@ stonith_construct_reply(xmlNode * request, const char *output, xmlNode * data, i - crm_xml_add(reply, "st_output", output); - crm_xml_add_int(reply, F_STONITH_RC, rc); - -- CRM_CHECK(request != NULL, crm_warn("Can't create a sane reply"); return reply); -- for (lpc = 0; lpc < DIMOF(names); lpc++) { -- name = names[lpc]; -- value = crm_element_value(request, name); -- crm_xml_add(reply, name, value); -- } -+ if (request == NULL) { -+ /* Most likely, this is the result of a stonith operation that was -+ * initiated before we came up. Unfortunately that means we lack enough -+ * information to provide clients with a full result. -+ * -+ * @TODO Maybe synchronize this information at start-up? -+ */ -+ crm_warn("Missing request information for client notifications for " -+ "operation with result %d (initiated before we came up?)", rc); - -- if (data != NULL) { -- crm_trace("Attaching reply output"); -- add_message_xml(reply, F_STONITH_CALLDATA, data); -+ } else { -+ const char *name = NULL; -+ const char *value = NULL; -+ -+ const char *names[] = { -+ F_STONITH_OPERATION, -+ F_STONITH_CALLID, -+ F_STONITH_CLIENTID, -+ F_STONITH_CLIENTNAME, -+ F_STONITH_REMOTE_OP_ID, -+ F_STONITH_CALLOPTS -+ }; -+ -+ crm_trace("Creating a result reply with%s reply output (rc=%d)", -+ (data? "" : "out"), rc); -+ for (int lpc = 0; lpc < DIMOF(names); lpc++) { -+ name = names[lpc]; -+ value = crm_element_value(request, name); -+ crm_xml_add(reply, name, value); -+ } -+ if (data != NULL) { -+ add_message_xml(reply, F_STONITH_CALLDATA, data); -+ } - } - return reply; - } --- -1.8.3.1 - diff --git a/SOURCES/012-ipc_fix.patch b/SOURCES/012-ipc_fix.patch deleted file mode 100644 index 234b215..0000000 --- a/SOURCES/012-ipc_fix.patch +++ /dev/null @@ -1,85 +0,0 @@ -From f7389ac6f67804f20393951462a59a0b505dfe03 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 21 Jul 2020 16:41:18 -0500 -Subject: [PATCH] Fix: executor: only send executor notifications to executor - clients - -This bug has existed since Pacemaker Remote was first implemented, but was -hidden until crm_node -l/-p was recently modified to go through controller IPC, -because other command-line IPC API clients either fire-and-forget IPC requests -or merely count replies, rather than parse the content of replies. - -Previously, when the executor sent notifications of results, it broadcast the -notification to all IPC clients. Normally this behavior makes sense, but for -the executor in particular, it may be running as pacemaker-remoted, in which -case its IPC clients include not only clients that connected to the executor -IPC, but clients that connected via proxy to other IPC APIs on the cluster node -hosting the remote connection. - -With crm_node -l/-p, this meant that it was possible for an executor API -notification to arrive while crm_node was waiting for a controller IPC reply. -It would not find the information it needed and would report a protocol -violation error. - -The fix is to send executor notifications only to executor clients. ---- - daemons/execd/execd_commands.c | 9 +++++++++ - daemons/execd/remoted_proxy.c | 5 +++++ - include/crm/common/ipc_internal.h | 5 +++-- - 3 files changed, 17 insertions(+), 2 deletions(-) - -diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c -index aaf2976..685fcc7 100644 ---- a/daemons/execd/execd_commands.c -+++ b/daemons/execd/execd_commands.c -@@ -507,6 +507,15 @@ send_client_notify(gpointer key, gpointer value, gpointer user_data) - crm_trace("Skipping notification to client without name"); - return; - } -+ if (is_set(client->flags, pcmk__client_to_proxy)) { -+ /* We only want to notify clients of the executor IPC API. If we are -+ * running as Pacemaker Remote, we may have clients proxied to other -+ * IPC services in the cluster, so skip those. -+ */ -+ crm_trace("Skipping executor API notification to %s IPC client", -+ client->name); -+ return; -+ } - - rc = lrmd_server_send_notify(client, update_msg); - if (rc == pcmk_rc_ok) { -diff --git a/daemons/execd/remoted_proxy.c b/daemons/execd/remoted_proxy.c -index dda7eed..5c58de4 100644 ---- a/daemons/execd/remoted_proxy.c -+++ b/daemons/execd/remoted_proxy.c -@@ -88,6 +88,11 @@ ipc_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid, const char *ipc - client->userdata = strdup(ipc_proxy->id); - client->name = crm_strdup_printf("proxy-%s-%d-%.8s", ipc_channel, client->pid, client->id); - -+ /* Allow remote executor to distinguish between proxied local clients and -+ * actual executor API clients -+ */ -+ set_bit(client->flags, pcmk__client_to_proxy); -+ - g_hash_table_insert(ipc_clients, client->id, client); - - msg = create_xml_node(NULL, T_LRMD_IPC_PROXY); -diff --git a/include/crm/common/ipc_internal.h b/include/crm/common/ipc_internal.h -index 6a1fcf3..91b3435 100644 ---- a/include/crm/common/ipc_internal.h -+++ b/include/crm/common/ipc_internal.h -@@ -121,8 +121,9 @@ struct pcmk__remote_s { - }; - - enum pcmk__client_flags { -- pcmk__client_proxied = 0x00001, /* ipc_proxy code only */ -- pcmk__client_privileged = 0x00002, /* root or cluster user */ -+ pcmk__client_proxied = (1 << 0), // Remote client behind proxy -+ pcmk__client_privileged = (1 << 1), // root or cluster user -+ pcmk__client_to_proxy = (1 << 2), // Local client to be proxied - }; - - struct pcmk__client_s { --- -1.8.3.1 - diff --git a/SOURCES/013-pacemakerd.patch b/SOURCES/013-pacemakerd.patch deleted file mode 100644 index 3478450..0000000 --- a/SOURCES/013-pacemakerd.patch +++ /dev/null @@ -1,1072 +0,0 @@ -From b49880467c18ade43cc283036949b686d1413118 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Tue, 14 Apr 2020 14:12:22 -0500 -Subject: [PATCH 1/5] Low: pacemakerd: remove unneeded IPC and CPG commands - -Now that the controller handles crm_node's -l/-p needs, pacemakerd no longer -needs to. Backward compatibility isn't an issue because pacemakerd IPC isn't -proxied for Pacemaker Remote, so only local clients are relevant, and the -pacemakerd IPC never had a C API, which means it was internal only. - -Without those commands, pacemakerd no longer needs to track a process mask, -connect to CPG, or maintain the peer cache. - -The only remaining need for the cluster layer is to use corosync CFG to tell -corosync to initiate or block shutdown. ---- - daemons/pacemakerd/pacemakerd.c | 283 ++---------------------------------- - daemons/pacemakerd/pacemakerd.h | 2 +- - daemons/pacemakerd/pcmkd_corosync.c | 7 +- - 3 files changed, 16 insertions(+), 276 deletions(-) - -diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c -index 64c30e2..0f7b2db 100644 ---- a/daemons/pacemakerd/pacemakerd.c -+++ b/daemons/pacemakerd/pacemakerd.c -@@ -36,15 +36,12 @@ - #include - #include - --static gboolean pcmk_quorate = FALSE; - static gboolean fatal_error = FALSE; - static GMainLoop *mainloop = NULL; - static bool global_keep_tracking = false; - - #define PCMK_PROCESS_CHECK_INTERVAL 5 - --static const char *local_name = NULL; --static uint32_t local_nodeid = 0; - static crm_trigger_t *shutdown_trigger = NULL; - static const char *pid_file = PCMK_RUN_DIR "/pacemaker.pid"; - -@@ -105,23 +102,6 @@ static pcmk_child_t pcmk_children[] = { - static gboolean check_active_before_startup_processes(gpointer user_data); - static int child_liveness(pcmk_child_t *child); - static gboolean start_child(pcmk_child_t * child); --static gboolean update_node_processes(uint32_t id, const char *uname, -- uint32_t procs); --void update_process_clients(pcmk__client_t *client); -- --static uint32_t --get_process_list(void) --{ -- int lpc = 0; -- uint32_t procs = crm_get_cluster_proc(); -- -- for (lpc = 0; lpc < SIZEOF(pcmk_children); lpc++) { -- if (pcmk_children[lpc].pid != 0) { -- procs |= pcmk_children[lpc].flag; -- } -- } -- return procs; --} - - static void - pcmk_process_exit(pcmk_child_t * child) -@@ -129,16 +109,6 @@ pcmk_process_exit(pcmk_child_t * child) - child->pid = 0; - child->active_before_startup = FALSE; - -- /* Broadcast the fact that one of our processes died ASAP -- * -- * Try to get some logging of the cause out first though -- * because we're probably about to get fenced -- * -- * Potentially do this only if respawn_count > N -- * to allow for local recovery -- */ -- update_node_processes(local_nodeid, NULL, get_process_list()); -- - child->respawn_count += 1; - if (child->respawn_count > MAX_RESPAWN) { - crm_err("Child respawn count exceeded by %s", child->name); -@@ -148,8 +118,6 @@ pcmk_process_exit(pcmk_child_t * child) - if (shutdown_trigger) { - /* resume step-wise shutdown (returned TRUE yields no parallelizing) */ - mainloop_set_trigger(shutdown_trigger); -- /* intended to speed up propagating expected lay-off of the daemons? */ -- update_node_processes(local_nodeid, NULL, get_process_list()); - - } else if (!child->respawn) { - /* nothing to do */ -@@ -341,7 +309,6 @@ start_child(pcmk_child_t * child) - crm_info("Forked child %lld for process %s%s", - (long long) child->pid, child->name, - use_valgrind ? " (valgrind enabled: " VALGRIND_BIN ")" : ""); -- update_node_processes(local_nodeid, NULL, get_process_list()); - return TRUE; - - } else { -@@ -492,7 +459,6 @@ pcmk_shutdown_worker(gpointer user_data) - } - } - -- /* send_cluster_id(); */ - crm_notice("Shutdown complete"); - - { -@@ -567,22 +533,12 @@ pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size) - pcmk_shutdown(15); - - } else if (crm_str_eq(task, CRM_OP_RM_NODE_CACHE, TRUE)) { -- /* Send to everyone */ -- struct iovec *iov; -- int id = 0; -- const char *name = NULL; -- -- crm_element_value_int(msg, XML_ATTR_ID, &id); -- name = crm_element_value(msg, XML_ATTR_UNAME); -- crm_notice("Instructing peers to remove references to node %s/%u", name, id); -- -- iov = calloc(1, sizeof(struct iovec)); -- iov->iov_base = dump_xml_unformatted(msg); -- iov->iov_len = 1 + strlen(iov->iov_base); -- send_cpg_iov(iov); -+ crm_trace("Ignoring IPC request to purge node " -+ "because peer cache is not used"); - - } else { -- update_process_clients(c); -+ crm_debug("Unrecognized IPC command '%s' sent to pacemakerd", -+ crm_str(task)); - } - - free_xml(msg); -@@ -618,113 +574,6 @@ struct qb_ipcs_service_handlers mcp_ipc_callbacks = { - .connection_destroyed = pcmk_ipc_destroy - }; - --static void --send_xml_to_client(gpointer key, gpointer value, gpointer user_data) --{ -- pcmk__ipc_send_xml((pcmk__client_t *) value, 0, (xmlNode *) user_data, -- crm_ipc_server_event); --} -- --/*! -- * \internal -- * \brief Send an XML message with process list of all known peers to client(s) -- * -- * \param[in] client Send message to this client, or all clients if NULL -- */ --void --update_process_clients(pcmk__client_t *client) --{ -- GHashTableIter iter; -- crm_node_t *node = NULL; -- xmlNode *update = create_xml_node(NULL, "nodes"); -- -- if (is_corosync_cluster()) { -- crm_xml_add_int(update, "quorate", pcmk_quorate); -- } -- -- g_hash_table_iter_init(&iter, crm_peer_cache); -- while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) { -- xmlNode *xml = create_xml_node(update, "node"); -- -- crm_xml_add_int(xml, "id", node->id); -- crm_xml_add(xml, "uname", node->uname); -- crm_xml_add(xml, "state", node->state); -- crm_xml_add_int(xml, "processes", node->processes); -- } -- -- if(client) { -- crm_trace("Sending process list to client %s", client->id); -- send_xml_to_client(NULL, client, update); -- -- } else { -- crm_trace("Sending process list to %d clients", -- pcmk__ipc_client_count()); -- pcmk__foreach_ipc_client(send_xml_to_client, update); -- } -- free_xml(update); --} -- --/*! -- * \internal -- * \brief Send a CPG message with local node's process list to all peers -- */ --static void --update_process_peers(void) --{ -- /* Do nothing for corosync-2 based clusters */ -- -- struct iovec *iov = calloc(1, sizeof(struct iovec)); -- -- CRM_ASSERT(iov); -- if (local_name) { -- iov->iov_base = crm_strdup_printf("", -- local_name, get_process_list()); -- } else { -- iov->iov_base = crm_strdup_printf("", -- get_process_list()); -- } -- iov->iov_len = strlen(iov->iov_base) + 1; -- crm_trace("Sending %s", (char*) iov->iov_base); -- send_cpg_iov(iov); --} -- --/*! -- * \internal -- * \brief Update a node's process list, notifying clients and peers if needed -- * -- * \param[in] id Node ID of affected node -- * \param[in] uname Uname of affected node -- * \param[in] procs Affected node's process list mask -- * -- * \return TRUE if the process list changed, FALSE otherwise -- */ --static gboolean --update_node_processes(uint32_t id, const char *uname, uint32_t procs) --{ -- gboolean changed = FALSE; -- crm_node_t *node = crm_get_peer(id, uname); -- -- if (procs != 0) { -- if (procs != node->processes) { -- crm_debug("Node %s now has process list: %.32x (was %.32x)", -- node->uname, procs, node->processes); -- node->processes = procs; -- changed = TRUE; -- -- /* If local node's processes have changed, notify clients/peers */ -- if (id == local_nodeid) { -- update_process_clients(NULL); -- update_process_peers(); -- } -- -- } else { -- crm_trace("Node %s still has process list: %.32x", node->uname, procs); -- } -- } -- return changed; --} -- -- - static pcmk__cli_option_t long_options[] = { - // long option, argument type, storage, short option, description, flags - { -@@ -1126,91 +975,6 @@ init_children_processes(void) - setenv("PCMK_respawned", "true", 1); - } - --static void --mcp_cpg_destroy(gpointer user_data) --{ -- crm_crit("Lost connection to cluster layer, shutting down"); -- crm_exit(CRM_EX_DISCONNECT); --} -- --/*! -- * \internal -- * \brief Process a CPG message (process list or manual peer cache removal) -- * -- * \param[in] handle CPG connection (ignored) -- * \param[in] groupName CPG group name (ignored) -- * \param[in] nodeid ID of affected node -- * \param[in] pid Process ID (ignored) -- * \param[in] msg CPG XML message -- * \param[in] msg_len Length of msg in bytes (ignored) -- */ --static void --mcp_cpg_deliver(cpg_handle_t handle, -- const struct cpg_name *groupName, -- uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len) --{ -- xmlNode *xml = string2xml(msg); -- const char *task = crm_element_value(xml, F_CRM_TASK); -- -- crm_trace("Received CPG message (%s): %.200s", -- (task? task : "process list"), (char*)msg); -- -- if (task == NULL) { -- if (nodeid == local_nodeid) { -- crm_debug("Ignoring message with local node's process list"); -- } else { -- uint32_t procs = 0; -- const char *uname = crm_element_value(xml, "uname"); -- -- crm_element_value_int(xml, "proclist", (int *)&procs); -- if (update_node_processes(nodeid, uname, procs)) { -- update_process_clients(NULL); -- } -- } -- -- } else if (crm_str_eq(task, CRM_OP_RM_NODE_CACHE, TRUE)) { -- int id = 0; -- const char *name = NULL; -- -- crm_element_value_int(xml, XML_ATTR_ID, &id); -- name = crm_element_value(xml, XML_ATTR_UNAME); -- reap_crm_member(id, name); -- } -- -- if (xml != NULL) { -- free_xml(xml); -- } --} -- --static void --mcp_cpg_membership(cpg_handle_t handle, -- const struct cpg_name *groupName, -- const struct cpg_address *member_list, size_t member_list_entries, -- const struct cpg_address *left_list, size_t left_list_entries, -- const struct cpg_address *joined_list, size_t joined_list_entries) --{ -- /* Update peer cache if needed */ -- pcmk_cpg_membership(handle, groupName, member_list, member_list_entries, -- left_list, left_list_entries, -- joined_list, joined_list_entries); -- -- /* Always broadcast our own presence after any membership change */ -- update_process_peers(); --} -- --static gboolean --mcp_quorum_callback(unsigned long long seq, gboolean quorate) --{ -- pcmk_quorate = quorate; -- return TRUE; --} -- --static void --mcp_quorum_destroy(gpointer user_data) --{ -- crm_info("connection lost"); --} -- - int - main(int argc, char **argv) - { -@@ -1226,7 +990,6 @@ main(int argc, char **argv) - struct rlimit cores; - crm_ipc_t *old_instance = NULL; - qb_ipcs_service_t *ipcs = NULL; -- static crm_cluster_t cluster; - - crm_log_preinit(NULL, argc, argv); - pcmk__set_cli_options(NULL, "[options]", long_options, -@@ -1397,7 +1160,7 @@ main(int argc, char **argv) - } - - /* Allows us to block shutdown */ -- if (cluster_connect_cfg(&local_nodeid) == FALSE) { -+ if (!cluster_connect_cfg()) { - crm_err("Couldn't connect to Corosync's CFG service"); - crm_exit(CRM_EX_PROTOCOL); - } -@@ -1417,34 +1180,13 @@ main(int argc, char **argv) - crm_exit(CRM_EX_FATAL); - }; - -- cluster.destroy = mcp_cpg_destroy; -- cluster.cpg.cpg_deliver_fn = mcp_cpg_deliver; -- cluster.cpg.cpg_confchg_fn = mcp_cpg_membership; -- -- crm_set_autoreap(FALSE); -+ mainloop_add_signal(SIGTERM, pcmk_shutdown); -+ mainloop_add_signal(SIGINT, pcmk_shutdown); - -- rc = pcmk_ok; -+ init_children_processes(); - -- if (cluster_connect_cpg(&cluster) == FALSE) { -- crm_err("Couldn't connect to Corosync's CPG service"); -- rc = -ENOPROTOOPT; -- -- } else if (cluster_connect_quorum(mcp_quorum_callback, mcp_quorum_destroy) -- == FALSE) { -- rc = -ENOTCONN; -- -- } else { -- local_name = get_local_node_name(); -- update_node_processes(local_nodeid, local_name, get_process_list()); -- -- mainloop_add_signal(SIGTERM, pcmk_shutdown); -- mainloop_add_signal(SIGINT, pcmk_shutdown); -- -- init_children_processes(); -- -- crm_notice("Pacemaker daemon successfully started and accepting connections"); -- g_main_loop_run(mainloop); -- } -+ crm_notice("Pacemaker daemon successfully started and accepting connections"); -+ g_main_loop_run(mainloop); - - if (ipcs) { - crm_trace("Closing IPC server"); -@@ -1453,9 +1195,6 @@ main(int argc, char **argv) - } - - g_main_loop_unref(mainloop); -- -- cluster_disconnect_cpg(&cluster); - cluster_disconnect_cfg(); -- -- crm_exit(crm_errno2exit(rc)); -+ crm_exit(CRM_EX_OK); - } -diff --git a/daemons/pacemakerd/pacemakerd.h b/daemons/pacemakerd/pacemakerd.h -index d66ab10..ac2d842 100644 ---- a/daemons/pacemakerd/pacemakerd.h -+++ b/daemons/pacemakerd/pacemakerd.h -@@ -22,7 +22,7 @@ - - gboolean mcp_read_config(void); - --gboolean cluster_connect_cfg(uint32_t * nodeid); -+gboolean cluster_connect_cfg(void); - gboolean cluster_disconnect_cfg(void); - - void pcmk_shutdown(int nsig); -diff --git a/daemons/pacemakerd/pcmkd_corosync.c b/daemons/pacemakerd/pcmkd_corosync.c -index ec74908..156f965 100644 ---- a/daemons/pacemakerd/pcmkd_corosync.c -+++ b/daemons/pacemakerd/pcmkd_corosync.c -@@ -93,13 +93,14 @@ cluster_disconnect_cfg(void) - } while(counter < max) - - gboolean --cluster_connect_cfg(uint32_t * nodeid) -+cluster_connect_cfg(void) - { - cs_error_t rc; - int fd = -1, retries = 0, rv; - uid_t found_uid = 0; - gid_t found_gid = 0; - pid_t found_pid = 0; -+ uint32_t nodeid; - - static struct mainloop_fd_callbacks cfg_fd_callbacks = { - .dispatch = pcmk_cfg_dispatch, -@@ -134,14 +135,14 @@ cluster_connect_cfg(uint32_t * nodeid) - } - - retries = 0; -- cs_repeat(retries, 30, rc = corosync_cfg_local_get(cfg_handle, nodeid)); -+ cs_repeat(retries, 30, rc = corosync_cfg_local_get(cfg_handle, &nodeid)); - - if (rc != CS_OK) { - crm_err("corosync cfg local_get error %d", rc); - goto bail; - } - -- crm_debug("Our nodeid: %d", *nodeid); -+ crm_debug("Our nodeid: %lu", (unsigned long) nodeid); - mainloop_add_fd("corosync-cfg", G_PRIORITY_DEFAULT, fd, &cfg_handle, &cfg_fd_callbacks); - - return TRUE; --- -1.8.3.1 - - -From 23ad3803a12189a369d188f3d3e606142cf16c52 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 2 Jul 2020 11:25:28 -0500 -Subject: [PATCH 2/5] Refactor: pacemakerd: functionize removing core file - limit - -... for readability and code isolation ---- - daemons/pacemakerd/pacemakerd.c | 50 ++++++++++++++++++++++++----------------- - 1 file changed, 29 insertions(+), 21 deletions(-) - -diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c -index 0f7b2db..ac308e7 100644 ---- a/daemons/pacemakerd/pacemakerd.c -+++ b/daemons/pacemakerd/pacemakerd.c -@@ -975,10 +975,37 @@ init_children_processes(void) - setenv("PCMK_respawned", "true", 1); - } - -+static void -+remove_core_file_limit(void) -+{ -+ struct rlimit cores; -+ int rc = getrlimit(RLIMIT_CORE, &cores); -+ -+ if (rc < 0) { -+ crm_perror(LOG_ERR, "Cannot determine current maximum core size."); -+ return; -+ } -+ -+ if ((cores.rlim_max == 0) && (geteuid() == 0)) { -+ cores.rlim_max = RLIM_INFINITY; -+ } else { -+ crm_info("Maximum core file size is %llu bytes", -+ (unsigned long long) cores.rlim_max); -+ } -+ cores.rlim_cur = cores.rlim_max; -+ -+ rc = setrlimit(RLIMIT_CORE, &cores); -+ if (rc < 0) { -+ crm_perror(LOG_ERR, -+ "Core file generation will remain disabled." -+ " Core files are an important diagnostic tool, so" -+ " please consider enabling them by default."); -+ } -+} -+ - int - main(int argc, char **argv) - { -- int rc; - int flag; - int argerr = 0; - -@@ -987,7 +1014,6 @@ main(int argc, char **argv) - - uid_t pcmk_uid = 0; - gid_t pcmk_gid = 0; -- struct rlimit cores; - crm_ipc_t *old_instance = NULL; - qb_ipcs_service_t *ipcs = NULL; - -@@ -1099,25 +1125,7 @@ main(int argc, char **argv) - PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES); - mainloop = g_main_loop_new(NULL, FALSE); - -- rc = getrlimit(RLIMIT_CORE, &cores); -- if (rc < 0) { -- crm_perror(LOG_ERR, "Cannot determine current maximum core size."); -- } else { -- if (cores.rlim_max == 0 && geteuid() == 0) { -- cores.rlim_max = RLIM_INFINITY; -- } else { -- crm_info("Maximum core file size is: %lu", (unsigned long)cores.rlim_max); -- } -- cores.rlim_cur = cores.rlim_max; -- -- rc = setrlimit(RLIMIT_CORE, &cores); -- if (rc < 0) { -- crm_perror(LOG_ERR, -- "Core file generation will remain disabled." -- " Core files are an important diagnostic tool, so" -- " please consider enabling them by default."); -- } -- } -+ remove_core_file_limit(); - - if (pcmk_daemon_user(&pcmk_uid, &pcmk_gid) < 0) { - crm_err("Cluster user %s does not exist, aborting Pacemaker startup", CRM_DAEMON_USER); --- -1.8.3.1 - - -From 40b0891dc92767aad8495121afcbd6e68fd3830a Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 2 Jul 2020 11:26:09 -0500 -Subject: [PATCH 3/5] Log: pacemakerd: improve messages - ---- - daemons/pacemakerd/pacemakerd.c | 22 +++++----- - daemons/pacemakerd/pcmkd_corosync.c | 85 +++++++++++++++++-------------------- - 2 files changed, 50 insertions(+), 57 deletions(-) - -diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c -index ac308e7..0f459c0 100644 ---- a/daemons/pacemakerd/pacemakerd.c -+++ b/daemons/pacemakerd/pacemakerd.c -@@ -13,6 +13,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -344,7 +345,7 @@ start_child(pcmk_child_t * child) - - // Drop root group access if not needed - if (!need_root_group && (setgid(gid) < 0)) { -- crm_perror(LOG_ERR, "Could not set group to %d", gid); -+ crm_warn("Could not set group to %d: %s", gid, strerror(errno)); - } - - /* Initialize supplementary groups to only those always granted to -@@ -356,7 +357,8 @@ start_child(pcmk_child_t * child) - } - - if (uid && setuid(uid) < 0) { -- crm_perror(LOG_ERR, "Could not set user to %d (%s)", uid, child->uid); -+ crm_warn("Could not set user to %s (id %d): %s", -+ child->uid, uid, strerror(errno)); - } - - pcmk__close_fds_in_child(true); -@@ -370,7 +372,7 @@ start_child(pcmk_child_t * child) - } else { - (void)execvp(child->command, opts_default); - } -- crm_perror(LOG_ERR, "FATAL: Cannot exec %s", child->command); -+ crm_crit("Could not execute %s: %s", child->command, strerror(errno)); - crm_exit(CRM_EX_FATAL); - } - return TRUE; /* never reached */ -@@ -527,8 +529,7 @@ pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size) - - task = crm_element_value(msg, F_CRM_TASK); - if (crm_str_eq(task, CRM_OP_QUIT, TRUE)) { -- /* Time to quit */ -- crm_notice("Shutting down in response to ticket %s (%s)", -+ crm_notice("Shutting down in response to IPC request %s from %s", - crm_element_value(msg, F_CRM_REFERENCE), crm_element_value(msg, F_CRM_ORIGIN)); - pcmk_shutdown(15); - -@@ -982,7 +983,8 @@ remove_core_file_limit(void) - int rc = getrlimit(RLIMIT_CORE, &cores); - - if (rc < 0) { -- crm_perror(LOG_ERR, "Cannot determine current maximum core size."); -+ crm_warn("Cannot determine current maximum core file size: %s", -+ strerror(errno)); - return; - } - -@@ -996,10 +998,8 @@ remove_core_file_limit(void) - - rc = setrlimit(RLIMIT_CORE, &cores); - if (rc < 0) { -- crm_perror(LOG_ERR, -- "Core file generation will remain disabled." -- " Core files are an important diagnostic tool, so" -- " please consider enabling them by default."); -+ crm_warn("Cannot raise system limit on core file size " -+ "(consider doing so manually)"); - } - } - -@@ -1108,7 +1108,6 @@ main(int argc, char **argv) - crm_ipc_destroy(old_instance); - - if (mcp_read_config() == FALSE) { -- crm_notice("Could not obtain corosync config data, exiting"); - crm_exit(CRM_EX_UNAVAILABLE); - } - -@@ -1169,7 +1168,6 @@ main(int argc, char **argv) - - /* Allows us to block shutdown */ - if (!cluster_connect_cfg()) { -- crm_err("Couldn't connect to Corosync's CFG service"); - crm_exit(CRM_EX_PROTOCOL); - } - -diff --git a/daemons/pacemakerd/pcmkd_corosync.c b/daemons/pacemakerd/pcmkd_corosync.c -index 156f965..6f19803 100644 ---- a/daemons/pacemakerd/pcmkd_corosync.c -+++ b/daemons/pacemakerd/pcmkd_corosync.c -@@ -28,7 +28,6 @@ - - #include /* PCMK__SPECIAL_PID* */ - --enum cluster_type_e stack = pcmk_cluster_unknown; - static corosync_cfg_handle_t cfg_handle; - - /* =::=::=::= CFG - Shutdown stuff =::=::=::= */ -@@ -63,9 +62,8 @@ pcmk_cfg_dispatch(gpointer user_data) - static void - cfg_connection_destroy(gpointer user_data) - { -- crm_err("Connection destroyed"); -+ crm_err("Lost connection to Corosync"); - cfg_handle = 0; -- - pcmk_shutdown(SIGTERM); - } - -@@ -85,7 +83,7 @@ cluster_disconnect_cfg(void) - code; \ - if(rc == CS_ERR_TRY_AGAIN || rc == CS_ERR_QUEUE_FULL) { \ - counter++; \ -- crm_debug("Retrying operation after %ds", counter); \ -+ crm_debug("Retrying Corosync operation after %ds", counter); \ - sleep(counter); \ - } else { \ - break; \ -@@ -110,41 +108,42 @@ cluster_connect_cfg(void) - cs_repeat(retries, 30, rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks)); - - if (rc != CS_OK) { -- crm_err("corosync cfg init: %s (%d)", cs_strerror(rc), rc); -+ crm_crit("Could not connect to Corosync CFG: %s " CRM_XS " rc=%d", -+ cs_strerror(rc), rc); - return FALSE; - } - - rc = corosync_cfg_fd_get(cfg_handle, &fd); - if (rc != CS_OK) { -- crm_err("corosync cfg fd_get: %s (%d)", cs_strerror(rc), rc); -+ crm_crit("Could not get Corosync CFG descriptor: %s " CRM_XS " rc=%d", -+ cs_strerror(rc), rc); - goto bail; - } - - /* CFG provider run as root (in given user namespace, anyway)? */ - if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, - &found_uid, &found_gid))) { -- crm_err("CFG provider is not authentic:" -- " process %lld (uid: %lld, gid: %lld)", -- (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -- (long long) found_uid, (long long) found_gid); -+ crm_crit("Rejecting Corosync CFG provider because process %lld " -+ "is running as uid %lld gid %lld, not root", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); - goto bail; - } else if (rv < 0) { -- crm_err("Could not verify authenticity of CFG provider: %s (%d)", -- strerror(-rv), -rv); -+ crm_crit("Could not authenticate Corosync CFG provider: %s " -+ CRM_XS " rc=%d", strerror(-rv), -rv); - goto bail; - } - - retries = 0; - cs_repeat(retries, 30, rc = corosync_cfg_local_get(cfg_handle, &nodeid)); -- - if (rc != CS_OK) { -- crm_err("corosync cfg local_get error %d", rc); -+ crm_crit("Could not get local node ID from Corosync: %s " -+ CRM_XS " rc=%d", cs_strerror(rc), rc); - goto bail; - } -+ crm_debug("Corosync reports local node ID is %lu", (unsigned long) nodeid); - -- crm_debug("Our nodeid: %lu", (unsigned long) nodeid); - mainloop_add_fd("corosync-cfg", G_PRIORITY_DEFAULT, fd, &cfg_handle, &cfg_fd_callbacks); -- - return TRUE; - - bail: -@@ -184,14 +183,15 @@ mcp_read_config(void) - gid_t found_gid = 0; - pid_t found_pid = 0; - int rv; -+ enum cluster_type_e stack; - - // There can be only one possibility - do { - rc = cmap_initialize(&local_handle); - if (rc != CS_OK) { - retries++; -- printf("cmap connection setup failed: %s. Retrying in %ds\n", cs_strerror(rc), retries); -- crm_info("cmap connection setup failed: %s. Retrying in %ds", cs_strerror(rc), retries); -+ crm_info("Could not connect to Corosync CMAP: %s (retrying in %ds) " -+ CRM_XS " rc=%d", cs_strerror(rc), retries, rc); - sleep(retries); - - } else { -@@ -201,15 +201,15 @@ mcp_read_config(void) - } while (retries < 5); - - if (rc != CS_OK) { -- printf("Could not connect to Cluster Configuration Database API, error %d\n", rc); -- crm_warn("Could not connect to Cluster Configuration Database API, error %d", rc); -+ crm_crit("Could not connect to Corosync CMAP: %s " -+ CRM_XS " rc=%d", cs_strerror(rc), rc); - return FALSE; - } - - rc = cmap_fd_get(local_handle, &fd); - if (rc != CS_OK) { -- crm_err("Could not obtain the CMAP API connection: %s (%d)", -- cs_strerror(rc), rc); -+ crm_crit("Could not get Corosync CMAP descriptor: %s " CRM_XS " rc=%d", -+ cs_strerror(rc), rc); - cmap_finalize(local_handle); - return FALSE; - } -@@ -217,38 +217,33 @@ mcp_read_config(void) - /* CMAP provider run as root (in given user namespace, anyway)? */ - if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, - &found_uid, &found_gid))) { -- crm_err("CMAP provider is not authentic:" -- " process %lld (uid: %lld, gid: %lld)", -- (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -- (long long) found_uid, (long long) found_gid); -+ crm_crit("Rejecting Corosync CMAP provider because process %lld " -+ "is running as uid %lld gid %lld, not root", -+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid), -+ (long long) found_uid, (long long) found_gid); - cmap_finalize(local_handle); - return FALSE; - } else if (rv < 0) { -- crm_err("Could not verify authenticity of CMAP provider: %s (%d)", -- strerror(-rv), -rv); -+ crm_crit("Could not authenticate Corosync CMAP provider: %s " -+ CRM_XS " rc=%d", strerror(-rv), -rv); - cmap_finalize(local_handle); - return FALSE; - } - - stack = get_cluster_type(); -- crm_info("Reading configure for stack: %s", name_for_cluster_type(stack)); -- -- /* =::=::= Should we be here =::=::= */ -- if (stack == pcmk_cluster_corosync) { -- pcmk__set_env_option("cluster_type", "corosync"); -- pcmk__set_env_option("quorum_type", "corosync"); -- -- } else { -- crm_err("Unsupported stack type: %s", name_for_cluster_type(stack)); -+ if (stack != pcmk_cluster_corosync) { -+ crm_crit("Expected corosync stack but detected %s " CRM_XS " stack=%d", -+ name_for_cluster_type(stack), stack); - return FALSE; - } - -- /* =::=::= Logging =::=::= */ -- if (pcmk__env_option("debug")) { -- /* Syslog logging is already setup by crm_log_init() */ -+ crm_info("Reading configuration for %s stack", -+ name_for_cluster_type(stack)); -+ pcmk__set_env_option("cluster_type", "corosync"); -+ pcmk__set_env_option("quorum_type", "corosync"); - -- } else { -- /* Check corosync */ -+ // If debug logging is not configured, check whether corosync has it -+ if (pcmk__env_option("debug") == NULL) { - char *debug_enabled = NULL; - - get_config_opt(config, local_handle, "logging.debug", &debug_enabled, "off"); -@@ -269,7 +264,7 @@ mcp_read_config(void) - if(local_handle){ - gid_t gid = 0; - if (pcmk_daemon_user(NULL, &gid) < 0) { -- crm_warn("Could not authorize group with corosync " CRM_XS -+ crm_warn("Could not authorize group with Corosync " CRM_XS - " No group found for user %s", CRM_DAEMON_USER); - - } else { -@@ -277,8 +272,8 @@ mcp_read_config(void) - snprintf(key, PATH_MAX, "uidgid.gid.%u", gid); - rc = cmap_set_uint8(local_handle, key, 1); - if (rc != CS_OK) { -- crm_warn("Could not authorize group with corosync "CRM_XS -- " group=%u rc=%d (%s)", gid, rc, ais_error2text(rc)); -+ crm_warn("Could not authorize group with Corosync: %s " CRM_XS -+ " group=%u rc=%d", ais_error2text(rc), gid, rc); - } - } - } --- -1.8.3.1 - - -From c1e12aeb58366f508730defcb8eb6f3ebedac469 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 23 Apr 2020 13:30:58 -0500 -Subject: [PATCH 4/5] Refactor: pacemakerd: use existing handle for corosync - shutdown - ---- - daemons/pacemakerd/pacemakerd.c | 31 ++++--------------------------- - daemons/pacemakerd/pacemakerd.h | 1 + - daemons/pacemakerd/pcmkd_corosync.c | 24 +++++++++++++++++++++++- - 3 files changed, 28 insertions(+), 28 deletions(-) - -diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c -index 0f459c0..fb35dfc 100644 ---- a/daemons/pacemakerd/pacemakerd.c -+++ b/daemons/pacemakerd/pacemakerd.c -@@ -30,10 +30,6 @@ - #include - #include - --#ifdef SUPPORT_COROSYNC --#include --#endif -- - #include - #include - -@@ -145,28 +141,6 @@ pcmk_process_exit(pcmk_child_t * child) - } - } - --static void pcmk_exit_with_cluster(int exitcode) --{ --#ifdef SUPPORT_COROSYNC -- corosync_cfg_handle_t cfg_handle; -- cs_error_t err; -- -- if (exitcode == CRM_EX_FATAL) { -- crm_info("Asking Corosync to shut down"); -- err = corosync_cfg_initialize(&cfg_handle, NULL); -- if (err != CS_OK) { -- crm_warn("Unable to open handle to corosync to close it down. err=%d", err); -- } -- err = corosync_cfg_try_shutdown(cfg_handle, COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE); -- if (err != CS_OK) { -- crm_warn("Corosync shutdown failed. err=%d", err); -- } -- corosync_cfg_finalize(cfg_handle); -- } --#endif -- crm_exit(exitcode); --} -- - static void - pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode) - { -@@ -475,7 +449,10 @@ pcmk_shutdown_worker(gpointer user_data) - - if (fatal_error) { - crm_notice("Shutting down and staying down after fatal error"); -- pcmk_exit_with_cluster(CRM_EX_FATAL); -+#ifdef SUPPORT_COROSYNC -+ pcmkd_shutdown_corosync(); -+#endif -+ crm_exit(CRM_EX_FATAL); - } - - return TRUE; -diff --git a/daemons/pacemakerd/pacemakerd.h b/daemons/pacemakerd/pacemakerd.h -index ac2d842..5f475fd 100644 ---- a/daemons/pacemakerd/pacemakerd.h -+++ b/daemons/pacemakerd/pacemakerd.h -@@ -24,5 +24,6 @@ gboolean mcp_read_config(void); - - gboolean cluster_connect_cfg(void); - gboolean cluster_disconnect_cfg(void); -+void pcmkd_shutdown_corosync(void); - - void pcmk_shutdown(int nsig); -diff --git a/daemons/pacemakerd/pcmkd_corosync.c b/daemons/pacemakerd/pcmkd_corosync.c -index 6f19803..82bd257 100644 ---- a/daemons/pacemakerd/pcmkd_corosync.c -+++ b/daemons/pacemakerd/pcmkd_corosync.c -@@ -28,7 +28,7 @@ - - #include /* PCMK__SPECIAL_PID* */ - --static corosync_cfg_handle_t cfg_handle; -+static corosync_cfg_handle_t cfg_handle = 0; - - /* =::=::=::= CFG - Shutdown stuff =::=::=::= */ - -@@ -151,6 +151,28 @@ cluster_connect_cfg(void) - return FALSE; - } - -+void -+pcmkd_shutdown_corosync(void) -+{ -+ cs_error_t rc; -+ -+ if (cfg_handle == 0) { -+ crm_warn("Unable to shut down Corosync: No connection"); -+ return; -+ } -+ crm_info("Asking Corosync to shut down"); -+ rc = corosync_cfg_try_shutdown(cfg_handle, -+ COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE); -+ if (rc == CS_OK) { -+ corosync_cfg_finalize(cfg_handle); -+ cfg_handle = 0; -+ } else { -+ crm_warn("Corosync shutdown failed: %s " CRM_XS " rc=%d", -+ cs_strerror(rc), rc); -+ } -+} -+ -+ - /* =::=::=::= Configuration =::=::=::= */ - static int - get_config_opt(uint64_t unused, cmap_handle_t object_handle, const char *key, char **value, --- -1.8.3.1 - - -From e93857b25a22045d5db7bd45c3f026fb82e6da8d Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 23 Apr 2020 13:36:08 -0500 -Subject: [PATCH 5/5] Build: pacemakerd: properly conditionalize corosync calls - -Previously, pacemakerd in its entirety was not built unless corosync support -was enabled, a throwback to when CMAN and Corosync 1 were supported. Now, -pacemakerd is built unconditionally, and just its corosync-related calls are -guarded by corosync support. - -This has no effect currently since corosync 2+ is the only supported cluster -layer, but it offers some future-proofing. ---- - daemons/pacemakerd/Makefile.am | 9 ++++----- - daemons/pacemakerd/pacemakerd.c | 6 ++++++ - 2 files changed, 10 insertions(+), 5 deletions(-) - -diff --git a/daemons/pacemakerd/Makefile.am b/daemons/pacemakerd/Makefile.am -index b01d8ef..4cc8a7c 100644 ---- a/daemons/pacemakerd/Makefile.am -+++ b/daemons/pacemakerd/Makefile.am -@@ -1,5 +1,5 @@ - # --# Copyright 2004-2019 the Pacemaker project contributors -+# Copyright 2004-2020 the Pacemaker project contributors - # - # The version control history for this file may have further details. - # -@@ -9,8 +9,6 @@ - - include $(top_srcdir)/mk/common.mk - --if BUILD_CS_SUPPORT -- - initdir = $(INITDIR) - init_SCRIPTS = pacemaker - sbin_PROGRAMS = pacemakerd -@@ -30,8 +28,9 @@ pacemakerd_LDFLAGS = $(LDFLAGS_HARDENED_EXE) - - pacemakerd_LDADD = $(top_builddir)/lib/cluster/libcrmcluster.la $(top_builddir)/lib/common/libcrmcommon.la - pacemakerd_LDADD += $(CLUSTERLIBS) --pacemakerd_SOURCES = pacemakerd.c pcmkd_corosync.c -- -+pacemakerd_SOURCES = pacemakerd.c -+if BUILD_CS_SUPPORT -+pacemakerd_SOURCES += pcmkd_corosync.c - endif - - CLEANFILES = $(man8_MANS) -diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c -index fb35dfc..652d6ca 100644 ---- a/daemons/pacemakerd/pacemakerd.c -+++ b/daemons/pacemakerd/pacemakerd.c -@@ -1084,9 +1084,11 @@ main(int argc, char **argv) - crm_ipc_close(old_instance); - crm_ipc_destroy(old_instance); - -+#ifdef SUPPORT_COROSYNC - if (mcp_read_config() == FALSE) { - crm_exit(CRM_EX_UNAVAILABLE); - } -+#endif - - // OCF shell functions and cluster-glue need facility under different name - { -@@ -1143,10 +1145,12 @@ main(int argc, char **argv) - crm_exit(CRM_EX_OSERR); - } - -+#ifdef SUPPORT_COROSYNC - /* Allows us to block shutdown */ - if (!cluster_connect_cfg()) { - crm_exit(CRM_EX_PROTOCOL); - } -+#endif - - if(pcmk_locate_sbd() > 0) { - setenv("PCMK_watchdog", "true", 1); -@@ -1178,6 +1182,8 @@ main(int argc, char **argv) - } - - g_main_loop_unref(mainloop); -+#ifdef SUPPORT_COROSYNC - cluster_disconnect_cfg(); -+#endif - crm_exit(CRM_EX_OK); - } --- -1.8.3.1 - diff --git a/SOURCES/014-sbd.patch b/SOURCES/014-sbd.patch deleted file mode 100644 index c830bfe..0000000 --- a/SOURCES/014-sbd.patch +++ /dev/null @@ -1,1187 +0,0 @@ -From 17d5ceac78f610aabf6a3678813706faf252c2fb Mon Sep 17 00:00:00 2001 -From: Klaus Wenninger -Date: Mon, 20 Jul 2020 17:56:29 +0200 -Subject: [PATCH 1/6] Fix: ipc-api: allow calling connect after disconnection - ---- - lib/common/crmcommon_private.h | 1 + - lib/common/ipc_client.c | 22 ++++++++++++++++------ - 2 files changed, 17 insertions(+), 6 deletions(-) - -diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h -index 49dae6c..d55df99 100644 ---- a/lib/common/crmcommon_private.h -+++ b/lib/common/crmcommon_private.h -@@ -175,6 +175,7 @@ typedef struct pcmk__ipc_methods_s { - struct pcmk_ipc_api_s { - enum pcmk_ipc_server server; // Daemon this IPC API instance is for - enum pcmk_ipc_dispatch dispatch_type; // How replies should be dispatched -+ size_t ipc_size_max; // maximum IPC buffer size - crm_ipc_t *ipc; // IPC connection - mainloop_io_t *mainloop_io; // If using mainloop, I/O source for IPC - bool free_on_disconnect; // Whether disconnect should free object -diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c -index 4077d61..df687da 100644 ---- a/lib/common/ipc_client.c -+++ b/lib/common/ipc_client.c -@@ -46,8 +46,6 @@ - int - pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server) - { -- size_t max_size = 0; -- - if (api == NULL) { - return EINVAL; - } -@@ -64,13 +62,15 @@ pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server) - return EOPNOTSUPP; - } - -+ (*api)->ipc_size_max = 0; -+ - // Set server methods and max_size (if not default) - switch (server) { - case pcmk_ipc_attrd: - break; - - case pcmk_ipc_based: -- max_size = 512 * 1024; // 512KB -+ (*api)->ipc_size_max = 512 * 1024; // 512KB - break; - - case pcmk_ipc_controld: -@@ -88,7 +88,7 @@ pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server) - - case pcmk_ipc_schedulerd: - // @TODO max_size could vary by client, maybe take as argument? -- max_size = 5 * 1024 * 1024; // 5MB -+ (*api)->ipc_size_max = 5 * 1024 * 1024; // 5MB - break; - } - if ((*api)->cmds == NULL) { -@@ -97,7 +97,8 @@ pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server) - return ENOMEM; - } - -- (*api)->ipc = crm_ipc_new(pcmk_ipc_name(*api, false), max_size); -+ (*api)->ipc = crm_ipc_new(pcmk_ipc_name(*api, false), -+ (*api)->ipc_size_max); - if ((*api)->ipc == NULL) { - pcmk_free_ipc_api(*api); - *api = NULL; -@@ -451,11 +452,20 @@ pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type) - { - int rc = pcmk_rc_ok; - -- if ((api == NULL) || (api->ipc == NULL)) { -+ if (api == NULL) { - crm_err("Cannot connect to uninitialized API object"); - return EINVAL; - } - -+ if (api->ipc == NULL) { -+ api->ipc = crm_ipc_new(pcmk_ipc_name(api, false), -+ api->ipc_size_max); -+ if (api->ipc == NULL) { -+ crm_err("Failed to re-create IPC API"); -+ return ENOMEM; -+ } -+ } -+ - if (crm_ipc_connected(api->ipc)) { - crm_trace("Already connected to %s IPC API", pcmk_ipc_name(api, true)); - return pcmk_rc_ok; --- -1.8.3.1 - - -From e5ad1a6c54da48c86c8ab262abd4921cb37e998d Mon Sep 17 00:00:00 2001 -From: Klaus Wenninger -Date: Mon, 20 Jul 2020 18:18:01 +0200 -Subject: [PATCH 2/6] Fix: ipc-api: avoid infinite loop when disconnected - -Happens when using pcmk_dispatch_ipc when dispatching without -mainloop. ---- - lib/common/ipc_client.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c -index df687da..aa032fe 100644 ---- a/lib/common/ipc_client.c -+++ b/lib/common/ipc_client.c -@@ -392,7 +392,7 @@ pcmk_dispatch_ipc(pcmk_ipc_api_t *api) - if (api == NULL) { - return; - } -- while (crm_ipc_ready(api->ipc)) { -+ while (crm_ipc_ready(api->ipc) > 0) { - if (crm_ipc_read(api->ipc) > 0) { - dispatch_ipc_data(crm_ipc_buffer(api->ipc), 0, api); - } --- -1.8.3.1 - - -From 927b43a57d5e8256fbce8fe0792f8ea228c57687 Mon Sep 17 00:00:00 2001 -From: Klaus Wenninger -Date: Mon, 9 Dec 2019 15:13:11 +0100 -Subject: [PATCH 3/6] Fix: sbd-integration: sync pacemakerd with sbd - -Make pacemakerd wait to be pinged by sbd before starting -sub-daemons. Pings further reply health-state and timestamp -of last successful check. On shutdown bring down all the -sub-daemons and wait to be polled for state by sbd before -finally exiting pacemakerd. -Add new api as not to make the xml-structure an external interface. ---- - daemons/pacemakerd/pacemakerd.c | 100 ++++++++++++++-- - include/crm/common/Makefile.am | 2 +- - include/crm/common/ipc_pacemakerd.h | 71 +++++++++++ - include/crm/msg_xml.h | 7 ++ - lib/common/Makefile.am | 1 + - lib/common/crmcommon_private.h | 3 + - lib/common/ipc_client.c | 5 +- - lib/common/ipc_pacemakerd.c | 232 ++++++++++++++++++++++++++++++++++++ - 8 files changed, 410 insertions(+), 11 deletions(-) - create mode 100644 include/crm/common/ipc_pacemakerd.h - create mode 100644 lib/common/ipc_pacemakerd.c - -diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c -index 652d6ca..ccfae66 100644 ---- a/daemons/pacemakerd/pacemakerd.c -+++ b/daemons/pacemakerd/pacemakerd.c -@@ -40,8 +40,25 @@ static bool global_keep_tracking = false; - #define PCMK_PROCESS_CHECK_INTERVAL 5 - - static crm_trigger_t *shutdown_trigger = NULL; -+static crm_trigger_t *startup_trigger = NULL; - static const char *pid_file = PCMK_RUN_DIR "/pacemaker.pid"; - -+/* state we report when asked via pacemakerd-api status-ping */ -+static const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT; -+static gboolean running_with_sbd = FALSE; /* local copy */ -+/* When contacted via pacemakerd-api by a client having sbd in -+ * the name we assume it is sbd-daemon which wants to know -+ * if pacemakerd shutdown gracefully. -+ * Thus when everything is shutdown properly pacemakerd -+ * waits till it has reported the graceful completion of -+ * shutdown to sbd and just when sbd-client closes the -+ * connection we can assume that the report has arrived -+ * properly so that pacemakerd can finally exit. -+ * Following two variables are used to track that handshake. -+ */ -+static unsigned int shutdown_complete_state_reported_to = 0; -+static gboolean shutdown_complete_state_reported_client_closed = FALSE; -+ - typedef struct pcmk_child_s { - pid_t pid; - long flag; -@@ -374,21 +391,20 @@ escalate_shutdown(gpointer data) - static gboolean - pcmk_shutdown_worker(gpointer user_data) - { -- static int phase = 0; -+ static int phase = SIZEOF(pcmk_children); - static time_t next_log = 0; -- static int max = SIZEOF(pcmk_children); - - int lpc = 0; - -- if (phase == 0) { -+ if (phase == SIZEOF(pcmk_children)) { - crm_notice("Shutting down Pacemaker"); -- phase = max; -+ pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN; - } - - for (; phase > 0; phase--) { - /* Don't stop anything with start_seq < 1 */ - -- for (lpc = max - 1; lpc >= 0; lpc--) { -+ for (lpc = SIZEOF(pcmk_children) - 1; lpc >= 0; lpc--) { - pcmk_child_t *child = &(pcmk_children[lpc]); - - if (phase != child->start_seq) { -@@ -436,6 +452,11 @@ pcmk_shutdown_worker(gpointer user_data) - } - - crm_notice("Shutdown complete"); -+ pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE; -+ if (!fatal_error && running_with_sbd && -+ !shutdown_complete_state_reported_client_closed) { -+ return TRUE; -+ } - - { - const char *delay = pcmk__env_option("shutdown_delay"); -@@ -489,6 +510,51 @@ pcmk_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid) - return 0; - } - -+static void -+pcmk_handle_ping_request(pcmk__client_t *c, xmlNode *msg, uint32_t id) -+{ -+ const char *value = NULL; -+ xmlNode *ping = NULL; -+ xmlNode *reply = NULL; -+ time_t pinged = time(NULL); -+ const char *from = crm_element_value(msg, F_CRM_SYS_FROM); -+ -+ /* Pinged for status */ -+ crm_trace("Pinged from %s.%s", -+ crm_str(crm_element_value(msg, F_CRM_ORIGIN)), -+ from?from:"unknown"); -+ ping = create_xml_node(NULL, XML_CRM_TAG_PING); -+ value = crm_element_value(msg, F_CRM_SYS_TO); -+ crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value); -+ crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state); -+ crm_xml_add_ll(ping, XML_ATTR_TSTAMP, (long long) pinged); -+ crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok"); -+ reply = create_reply(msg, ping); -+ free_xml(ping); -+ if (reply) { -+ if (pcmk__ipc_send_xml(c, id, reply, crm_ipc_server_event) != -+ pcmk_rc_ok) { -+ crm_err("Failed sending ping-reply"); -+ } -+ free_xml(reply); -+ } else { -+ crm_err("Failed building ping-reply"); -+ } -+ /* just proceed state on sbd pinging us */ -+ if (from && strstr(from, "sbd")) { -+ if (crm_str_eq(pacemakerd_state, -+ XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE, -+ TRUE)) { -+ shutdown_complete_state_reported_to = c->pid; -+ } else if (crm_str_eq(pacemakerd_state, -+ XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, -+ TRUE)) { -+ pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS; -+ mainloop_set_trigger(startup_trigger); -+ } -+ } -+} -+ - /* Exit code means? */ - static int32_t - pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size) -@@ -514,6 +580,9 @@ pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size) - crm_trace("Ignoring IPC request to purge node " - "because peer cache is not used"); - -+ } else if (crm_str_eq(task, CRM_OP_PING, TRUE)) { -+ pcmk_handle_ping_request(c, msg, id); -+ - } else { - crm_debug("Unrecognized IPC command '%s' sent to pacemakerd", - crm_str(task)); -@@ -533,6 +602,12 @@ pcmk_ipc_closed(qb_ipcs_connection_t * c) - return 0; - } - crm_trace("Connection %p", c); -+ if (shutdown_complete_state_reported_to == client->pid) { -+ shutdown_complete_state_reported_client_closed = TRUE; -+ if (shutdown_trigger) { -+ mainloop_set_trigger(shutdown_trigger); -+ } -+ } - pcmk__free_client(client); - return 0; - } -@@ -924,8 +999,8 @@ find_and_track_existing_processes(void) - return pcmk_rc_ok; - } - --static void --init_children_processes(void) -+static gboolean -+init_children_processes(void *user_data) - { - int start_seq = 1, lpc = 0; - static int max = SIZEOF(pcmk_children); -@@ -951,6 +1026,8 @@ init_children_processes(void) - * This may be useful for the daemons to know - */ - setenv("PCMK_respawned", "true", 1); -+ pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING; -+ return TRUE; - } - - static void -@@ -1154,6 +1231,7 @@ main(int argc, char **argv) - - if(pcmk_locate_sbd() > 0) { - setenv("PCMK_watchdog", "true", 1); -+ running_with_sbd = TRUE; - } else { - setenv("PCMK_watchdog", "false", 1); - } -@@ -1170,7 +1248,13 @@ main(int argc, char **argv) - mainloop_add_signal(SIGTERM, pcmk_shutdown); - mainloop_add_signal(SIGINT, pcmk_shutdown); - -- init_children_processes(); -+ if (running_with_sbd) { -+ pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING; -+ startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL); -+ } else { -+ pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS; -+ init_children_processes(NULL); -+ } - - crm_notice("Pacemaker daemon successfully started and accepting connections"); - g_main_loop_run(mainloop); -diff --git a/include/crm/common/Makefile.am b/include/crm/common/Makefile.am -index f29d105..1b5730a 100644 ---- a/include/crm/common/Makefile.am -+++ b/include/crm/common/Makefile.am -@@ -12,7 +12,7 @@ MAINTAINERCLEANFILES = Makefile.in - headerdir=$(pkgincludedir)/crm/common - - header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h results.h \ -- nvpair.h acl.h ipc_controld.h -+ nvpair.h acl.h ipc_controld.h ipc_pacemakerd.h - noinst_HEADERS = internal.h alerts_internal.h \ - iso8601_internal.h remote_internal.h xml_internal.h \ - ipc_internal.h output.h cmdline_internal.h curses_internal.h \ -diff --git a/include/crm/common/ipc_pacemakerd.h b/include/crm/common/ipc_pacemakerd.h -new file mode 100644 -index 0000000..00e3edd ---- /dev/null -+++ b/include/crm/common/ipc_pacemakerd.h -@@ -0,0 +1,71 @@ -+/* -+ * Copyright 2020 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. -+ * -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -+ */ -+ -+#ifndef PCMK__IPC_PACEMAKERD__H -+# define PCMK__IPC_PACEMAKERD__H -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+/** -+ * \file -+ * \brief IPC commands for Pacemakerd -+ * -+ * \ingroup core -+ */ -+ -+#include // time_t -+#include // pcmk_ipc_api_t -+ -+enum pcmk_pacemakerd_state { -+ pcmk_pacemakerd_state_invalid = -1, -+ pcmk_pacemakerd_state_init = 0, -+ pcmk_pacemakerd_state_starting_daemons, -+ pcmk_pacemakerd_state_wait_for_ping, -+ pcmk_pacemakerd_state_running, -+ pcmk_pacemakerd_state_shutting_down, -+ pcmk_pacemakerd_state_shutdown_complete, -+ pcmk_pacemakerd_state_max = pcmk_pacemakerd_state_shutdown_complete, -+}; -+ -+//! Possible types of pacemakerd replies -+enum pcmk_pacemakerd_api_reply { -+ pcmk_pacemakerd_reply_unknown, -+ pcmk_pacemakerd_reply_ping, -+}; -+ -+/*! -+ * Pacemakerd reply passed to event callback -+ */ -+typedef struct { -+ enum pcmk_pacemakerd_api_reply reply_type; -+ -+ union { -+ // pcmk_pacemakerd_reply_ping -+ struct { -+ const char *sys_from; -+ enum pcmk_pacemakerd_state state; -+ time_t last_good; -+ int status; -+ } ping; -+ } data; -+} pcmk_pacemakerd_api_reply_t; -+ -+int pcmk_pacemakerd_api_ping(pcmk_ipc_api_t *api, const char *ipc_name); -+enum pcmk_pacemakerd_state -+ pcmk_pacemakerd_api_daemon_state_text2enum(const char *state); -+const char -+ *pcmk_pacemakerd_api_daemon_state_enum2text(enum pcmk_pacemakerd_state state); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif // PCMK__IPC_PACEMAKERD__H -diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h -index af3f33e..1fcb72d 100644 ---- a/include/crm/msg_xml.h -+++ b/include/crm/msg_xml.h -@@ -123,6 +123,13 @@ extern "C" { - # define XML_PING_ATTR_STATUS "result" - # define XML_PING_ATTR_SYSFROM "crm_subsystem" - # define XML_PING_ATTR_CRMDSTATE "crmd_state" -+# define XML_PING_ATTR_PACEMAKERDSTATE "pacemakerd_state" -+# define XML_PING_ATTR_PACEMAKERDSTATE_INIT "init" -+# define XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS "starting_daemons" -+# define XML_PING_ATTR_PACEMAKERDSTATE_WAITPING "wait_for_ping" -+# define XML_PING_ATTR_PACEMAKERDSTATE_RUNNING "running" -+# define XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN "shutting_down" -+# define XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE "shutdown_complete" - - # define XML_TAG_FRAGMENT "cib_fragment" - -diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am -index db66a6e..e0249b9 100644 ---- a/lib/common/Makefile.am -+++ b/lib/common/Makefile.am -@@ -50,6 +50,7 @@ libcrmcommon_la_SOURCES += io.c - libcrmcommon_la_SOURCES += ipc_client.c - libcrmcommon_la_SOURCES += ipc_common.c - libcrmcommon_la_SOURCES += ipc_controld.c -+libcrmcommon_la_SOURCES += ipc_pacemakerd.c - libcrmcommon_la_SOURCES += ipc_server.c - libcrmcommon_la_SOURCES += iso8601.c - libcrmcommon_la_SOURCES += logging.c -diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h -index d55df99..68e3390 100644 ---- a/lib/common/crmcommon_private.h -+++ b/lib/common/crmcommon_private.h -@@ -210,4 +210,7 @@ bool pcmk__valid_ipc_header(const pcmk__ipc_header_t *header); - G_GNUC_INTERNAL - pcmk__ipc_methods_t *pcmk__controld_api_methods(void); - -+G_GNUC_INTERNAL -+pcmk__ipc_methods_t *pcmk__pacemakerd_api_methods(void); -+ - #endif // CRMCOMMON_PRIVATE__H -diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c -index aa032fe..033199d 100644 ---- a/lib/common/ipc_client.c -+++ b/lib/common/ipc_client.c -@@ -41,7 +41,7 @@ - * - * \note The caller is responsible for freeing *api using pcmk_free_ipc_api(). - * \note This is intended to supersede crm_ipc_new() but currently only -- * supports the controller IPC API. -+ * supports the controller & pacemakerd IPC API. - */ - int - pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server) -@@ -84,6 +84,7 @@ pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server) - break; - - case pcmk_ipc_pacemakerd: -+ (*api)->cmds = pcmk__pacemakerd_api_methods(); - break; - - case pcmk_ipc_schedulerd: -@@ -259,7 +260,7 @@ pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log) - return for_log? "fencer" : NULL /* "stonith-ng" */; - - case pcmk_ipc_pacemakerd: -- return for_log? "launcher" : NULL /* CRM_SYSTEM_MCP */; -+ return for_log? "launcher" : CRM_SYSTEM_MCP; - - case pcmk_ipc_schedulerd: - return for_log? "scheduler" : NULL /* CRM_SYSTEM_PENGINE */; -diff --git a/lib/common/ipc_pacemakerd.c b/lib/common/ipc_pacemakerd.c -new file mode 100644 -index 0000000..241722e ---- /dev/null -+++ b/lib/common/ipc_pacemakerd.c -@@ -0,0 +1,232 @@ -+/* -+ * Copyright 2020 the Pacemaker project contributors -+ * -+ * The version control history for this file may have further details. -+ * -+ * This source code is licensed under the GNU Lesser General Public License -+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. -+ */ -+ -+#include -+ -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include "crmcommon_private.h" -+ -+typedef struct pacemakerd_api_private_s { -+ enum pcmk_pacemakerd_state state; -+ char *client_uuid; -+} pacemakerd_api_private_t; -+ -+static const char *pacemakerd_state_str[] = { -+ XML_PING_ATTR_PACEMAKERDSTATE_INIT, -+ XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS, -+ XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, -+ XML_PING_ATTR_PACEMAKERDSTATE_RUNNING, -+ XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN, -+ XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE -+}; -+ -+enum pcmk_pacemakerd_state -+pcmk_pacemakerd_api_daemon_state_text2enum(const char *state) -+{ -+ int i; -+ -+ if (state == NULL) { -+ return pcmk_pacemakerd_state_invalid; -+ } -+ for (i=pcmk_pacemakerd_state_init; i <= pcmk_pacemakerd_state_max; -+ i++) { -+ if (crm_str_eq(state, pacemakerd_state_str[i], TRUE)) { -+ return i; -+ } -+ } -+ return pcmk_pacemakerd_state_invalid; -+} -+ -+const char * -+pcmk_pacemakerd_api_daemon_state_enum2text( -+ enum pcmk_pacemakerd_state state) -+{ -+ if ((state >= pcmk_pacemakerd_state_init) && -+ (state <= pcmk_pacemakerd_state_max)) { -+ return pacemakerd_state_str[state]; -+ } -+ return "invalid"; -+} -+ -+// \return Standard Pacemaker return code -+static int -+new_data(pcmk_ipc_api_t *api) -+{ -+ struct pacemakerd_api_private_s *private = NULL; -+ -+ api->api_data = calloc(1, sizeof(struct pacemakerd_api_private_s)); -+ -+ if (api->api_data == NULL) { -+ return errno; -+ } -+ -+ private = api->api_data; -+ private->state = pcmk_pacemakerd_state_invalid; -+ /* other as with cib, controld, ... we are addressing pacemakerd just -+ from the local node -> pid is unique and thus sufficient as an ID -+ */ -+ private->client_uuid = pcmk__getpid_s(); -+ -+ return pcmk_rc_ok; -+} -+ -+static void -+free_data(void *data) -+{ -+ free(((struct pacemakerd_api_private_s *) data)->client_uuid); -+ free(data); -+} -+ -+// \return Standard Pacemaker return code -+static int -+post_connect(pcmk_ipc_api_t *api) -+{ -+ struct pacemakerd_api_private_s *private = NULL; -+ -+ if (api->api_data == NULL) { -+ return EINVAL; -+ } -+ private = api->api_data; -+ private->state = pcmk_pacemakerd_state_invalid; -+ -+ return pcmk_rc_ok; -+} -+ -+static void -+post_disconnect(pcmk_ipc_api_t *api) -+{ -+ struct pacemakerd_api_private_s *private = NULL; -+ -+ if (api->api_data == NULL) { -+ return; -+ } -+ private = api->api_data; -+ private->state = pcmk_pacemakerd_state_invalid; -+ -+ return; -+} -+ -+static bool -+reply_expected(pcmk_ipc_api_t *api, xmlNode *request) -+{ -+ const char *command = crm_element_value(request, F_CRM_TASK); -+ -+ if (command == NULL) { -+ return false; -+ } -+ -+ // We only need to handle commands that functions in this file can send -+ return !strcmp(command, CRM_OP_PING); -+} -+ -+static void -+dispatch(pcmk_ipc_api_t *api, xmlNode *reply) -+{ -+ crm_exit_t status = CRM_EX_OK; -+ xmlNode *msg_data = NULL; -+ pcmk_pacemakerd_api_reply_t reply_data = { -+ pcmk_pacemakerd_reply_unknown -+ }; -+ const char *value = NULL; -+ long long value_ll = 0; -+ -+ value = crm_element_value(reply, F_CRM_MSG_TYPE); -+ if ((value == NULL) || (strcmp(value, XML_ATTR_RESPONSE))) { -+ crm_debug("Unrecognizable pacemakerd message: invalid message type '%s'", -+ crm_str(value)); -+ status = CRM_EX_PROTOCOL; -+ goto done; -+ } -+ -+ if (crm_element_value(reply, XML_ATTR_REFERENCE) == NULL) { -+ crm_debug("Unrecognizable pacemakerd message: no reference"); -+ status = CRM_EX_PROTOCOL; -+ goto done; -+ } -+ -+ value = crm_element_value(reply, F_CRM_TASK); -+ if ((value == NULL) || strcmp(value, CRM_OP_PING)) { -+ crm_debug("Unrecognizable pacemakerd message: '%s'", crm_str(value)); -+ status = CRM_EX_PROTOCOL; -+ goto done; -+ } -+ -+ // Parse useful info from reply -+ -+ msg_data = get_message_xml(reply, F_CRM_DATA); -+ crm_element_value_ll(msg_data, XML_ATTR_TSTAMP, &value_ll); -+ -+ reply_data.reply_type = pcmk_pacemakerd_reply_ping; -+ reply_data.data.ping.state = -+ pcmk_pacemakerd_api_daemon_state_text2enum( -+ crm_element_value(msg_data, XML_PING_ATTR_PACEMAKERDSTATE)); -+ reply_data.data.ping.status = -+ crm_str_eq(crm_element_value(msg_data, XML_PING_ATTR_STATUS), -+ "ok", FALSE)?pcmk_rc_ok:pcmk_rc_error; -+ reply_data.data.ping.last_good = (time_t) value_ll; -+ reply_data.data.ping.sys_from = crm_element_value(msg_data, -+ XML_PING_ATTR_SYSFROM); -+ -+done: -+ pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data); -+} -+ -+pcmk__ipc_methods_t * -+pcmk__pacemakerd_api_methods() -+{ -+ pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t)); -+ -+ if (cmds != NULL) { -+ cmds->new_data = new_data; -+ cmds->free_data = free_data; -+ cmds->post_connect = post_connect; -+ cmds->reply_expected = reply_expected; -+ cmds->dispatch = dispatch; -+ cmds->post_disconnect = post_disconnect; -+ } -+ return cmds; -+} -+ -+int -+pcmk_pacemakerd_api_ping(pcmk_ipc_api_t *api, const char *ipc_name) -+{ -+ pacemakerd_api_private_t *private; -+ xmlNode *cmd; -+ int rc; -+ -+ CRM_CHECK(api != NULL, return -EINVAL); -+ private = api->api_data; -+ CRM_ASSERT(private != NULL); -+ -+ cmd = create_request(CRM_OP_PING, NULL, NULL, CRM_SYSTEM_MCP, -+ ipc_name?ipc_name:((crm_system_name? crm_system_name : "client")), -+ private->client_uuid); -+ -+ if (cmd) { -+ rc = pcmk__send_ipc_request(api, cmd); -+ if (rc != pcmk_rc_ok) { -+ crm_debug("Couldn't ping pacemakerd: %s rc=%d", -+ pcmk_rc_str(rc), rc); -+ rc = ECOMM; -+ } -+ free_xml(cmd); -+ } else { -+ rc = ENOMSG; -+ } -+ -+ return rc; -+} --- -1.8.3.1 - - -From 06da3c3685b0bdf093a13067cc399e782115e39c Mon Sep 17 00:00:00 2001 -From: Klaus Wenninger -Date: Mon, 20 Jul 2020 23:28:32 +0200 -Subject: [PATCH 4/6] Feature: tools: Add -P to crmadmin to ping via - pacemakerd-api - ---- - include/crm/crm.h | 2 +- - tools/crmadmin.c | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++---- - 2 files changed, 152 insertions(+), 11 deletions(-) - -diff --git a/include/crm/crm.h b/include/crm/crm.h -index dc2adc1..ce2074b 100644 ---- a/include/crm/crm.h -+++ b/include/crm/crm.h -@@ -51,7 +51,7 @@ extern "C" { - * >=3.0.13: Fail counts include operation name and interval - * >=3.2.0: DC supports PCMK_LRM_OP_INVALID and PCMK_LRM_OP_NOT_CONNECTED - */ --# define CRM_FEATURE_SET "3.4.0" -+# define CRM_FEATURE_SET "3.4.1" - - # define EOS '\0' - # define DIMOF(a) ((int) (sizeof(a)/sizeof(a[0])) ) -diff --git a/tools/crmadmin.c b/tools/crmadmin.c -index 4688458..2ebdd14 100644 ---- a/tools/crmadmin.c -+++ b/tools/crmadmin.c -@@ -20,7 +20,9 @@ - #include - #include - #include -+#include - #include -+#include - #include - - #define DEFAULT_MESSAGE_TIMEOUT_MS 30000 -@@ -31,6 +33,8 @@ static GMainLoop *mainloop = NULL; - - bool do_work(pcmk_ipc_api_t *api); - void do_find_node_list(xmlNode *xml_node); -+static char *ipc_name = NULL; -+ - gboolean admin_message_timeout(gpointer data); - - static enum { -@@ -40,6 +44,7 @@ static enum { - cmd_elect_dc, - cmd_whois_dc, - cmd_list_nodes, -+ cmd_pacemakerd_health, - } command = cmd_none; - - static gboolean BE_VERBOSE = FALSE; -@@ -82,6 +87,15 @@ static pcmk__cli_option_t long_options[] = { - pcmk__option_default - }, - { -+ "pacemakerd", no_argument, NULL, 'P', -+ "Display the status of local pacemakerd.", pcmk__option_default -+ }, -+ { -+ "-spacer-", no_argument, NULL, '-', -+ "\n\tResult is the state of the sub-daemons watched by pacemakerd.\n", -+ pcmk__option_default -+ }, -+ { - "dc_lookup", no_argument, NULL, 'D', - "Display the uname of the node co-ordinating the cluster.", - pcmk__option_default -@@ -122,16 +136,21 @@ static pcmk__cli_option_t long_options[] = { - { - "bash-export", no_argument, NULL, 'B', - "Display nodes as shell commands of the form 'export uname=uuid' " -- "(valid with -N/--nodes)'\n", -+ "(valid with -N/--nodes)", -+ pcmk__option_default -+ }, -+ { -+ "ipc-name", required_argument, NULL, 'i', -+ "Name to use for ipc instead of 'crmadmin' (with -P/--pacemakerd).", - pcmk__option_default - }, - { - "-spacer-", no_argument, NULL, '-', -- "Notes:", pcmk__option_default -+ "\nNotes:", pcmk__option_default - }, - { - "-spacer-", no_argument, NULL, '-', -- "The -K and -E commands do not work and may be removed in a future " -+ "\nThe -K and -E commands do not work and may be removed in a future " - "version.", - pcmk__option_default - }, -@@ -223,6 +242,88 @@ done: - quit_main_loop(exit_code); - } - -+static void -+pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api, -+ enum pcmk_ipc_event event_type, crm_exit_t status, -+ void *event_data, void *user_data) -+{ -+ pcmk_pacemakerd_api_reply_t *reply = event_data; -+ -+ switch (event_type) { -+ case pcmk_ipc_event_disconnect: -+ if (exit_code == CRM_EX_DISCONNECT) { // Unexpected -+ fprintf(stderr, "error: Lost connection to pacemakerd\n"); -+ } -+ goto done; -+ break; -+ -+ case pcmk_ipc_event_reply: -+ break; -+ -+ default: -+ return; -+ } -+ -+ if (message_timer_id != 0) { -+ g_source_remove(message_timer_id); -+ message_timer_id = 0; -+ } -+ -+ if (status != CRM_EX_OK) { -+ fprintf(stderr, "error: Bad reply from pacemakerd: %s", -+ crm_exit_str(status)); -+ exit_code = status; -+ goto done; -+ } -+ -+ if (reply->reply_type != pcmk_pacemakerd_reply_ping) { -+ fprintf(stderr, "error: Unknown reply type %d from pacemakerd\n", -+ reply->reply_type); -+ goto done; -+ } -+ -+ // Parse desired information from reply -+ switch (command) { -+ case cmd_pacemakerd_health: -+ { -+ crm_time_t *crm_when = crm_time_new(NULL); -+ char *pinged_buf = NULL; -+ -+ crm_time_set_timet(crm_when, &reply->data.ping.last_good); -+ pinged_buf = crm_time_as_string(crm_when, -+ crm_time_log_date | crm_time_log_timeofday | -+ crm_time_log_with_timezone); -+ -+ printf("Status of %s: '%s' %s %s\n", -+ reply->data.ping.sys_from, -+ (reply->data.ping.status == pcmk_rc_ok)? -+ pcmk_pacemakerd_api_daemon_state_enum2text( -+ reply->data.ping.state):"query failed", -+ (reply->data.ping.status == pcmk_rc_ok)?"last updated":"", -+ (reply->data.ping.status == pcmk_rc_ok)?pinged_buf:""); -+ if (BE_SILENT && -+ (reply->data.ping.state != pcmk_pacemakerd_state_invalid)) { -+ fprintf(stderr, "%s\n", -+ (reply->data.ping.status == pcmk_rc_ok)? -+ pcmk_pacemakerd_api_daemon_state_enum2text( -+ reply->data.ping.state): -+ "query failed"); -+ } -+ exit_code = CRM_EX_OK; -+ free(pinged_buf); -+ } -+ break; -+ -+ default: // Not really possible here -+ exit_code = CRM_EX_SOFTWARE; -+ break; -+ } -+ -+done: -+ pcmk_disconnect_ipc(pacemakerd_api); -+ quit_main_loop(exit_code); -+} -+ - // \return Standard Pacemaker return code - static int - list_nodes() -@@ -257,7 +358,9 @@ main(int argc, char **argv) - int flag; - int rc; - pcmk_ipc_api_t *controld_api = NULL; -+ pcmk_ipc_api_t *pacemakerd_api = NULL; - bool need_controld_api = true; -+ bool need_pacemakerd_api = false; - - crm_log_cli_init("crmadmin"); - pcmk__set_cli_options(NULL, " [options]", long_options, -@@ -282,7 +385,9 @@ main(int argc, char **argv) - message_timeout_ms = DEFAULT_MESSAGE_TIMEOUT_MS; - } - break; -- -+ case 'i': -+ ipc_name = strdup(optarg); -+ break; - case '$': - case '?': - pcmk__cli_help(flag, CRM_EX_OK); -@@ -304,6 +409,11 @@ main(int argc, char **argv) - case 'q': - BE_SILENT = TRUE; - break; -+ case 'P': -+ command = cmd_pacemakerd_health; -+ need_pacemakerd_api = true; -+ need_controld_api = false; -+ break; - case 'S': - command = cmd_health; - crm_trace("Option %c => %s", flag, optarg); -@@ -369,7 +479,26 @@ main(int argc, char **argv) - } - } - -- if (do_work(controld_api)) { -+ // Connect to pacemakerd if needed -+ if (need_pacemakerd_api) { -+ rc = pcmk_new_ipc_api(&pacemakerd_api, pcmk_ipc_pacemakerd); -+ if (pacemakerd_api == NULL) { -+ fprintf(stderr, "error: Could not connect to pacemakerd: %s\n", -+ pcmk_rc_str(rc)); -+ exit_code = pcmk_rc2exitc(rc); -+ goto done; -+ } -+ pcmk_register_ipc_callback(pacemakerd_api, pacemakerd_event_cb, NULL); -+ rc = pcmk_connect_ipc(pacemakerd_api, pcmk_ipc_dispatch_main); -+ if (rc != pcmk_rc_ok) { -+ fprintf(stderr, "error: Could not connect to pacemakerd: %s\n", -+ pcmk_rc_str(rc)); -+ exit_code = pcmk_rc2exitc(rc); -+ goto done; -+ } -+ } -+ -+ if (do_work(controld_api?controld_api:pacemakerd_api)) { - // A reply is needed from controller, so run main loop to get it - exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects - mainloop = g_main_loop_new(NULL, FALSE); -@@ -379,12 +508,19 @@ main(int argc, char **argv) - } - - done: -+ - if (controld_api != NULL) { - pcmk_ipc_api_t *capi = controld_api; -- - controld_api = NULL; // Ensure we can't free this twice - pcmk_free_ipc_api(capi); - } -+ -+ if (pacemakerd_api != NULL) { -+ pcmk_ipc_api_t *capi = pacemakerd_api; -+ pacemakerd_api = NULL; // Ensure we can't free this twice -+ pcmk_free_ipc_api(capi); -+ } -+ - if (mainloop != NULL) { - g_main_loop_unref(mainloop); - mainloop = NULL; -@@ -394,30 +530,35 @@ done: - - // \return True if reply from controller is needed - bool --do_work(pcmk_ipc_api_t *controld_api) -+do_work(pcmk_ipc_api_t *api) - { - bool need_reply = false; - int rc = pcmk_rc_ok; - - switch (command) { - case cmd_shutdown: -- rc = pcmk_controld_api_shutdown(controld_api, dest_node); -+ rc = pcmk_controld_api_shutdown(api, dest_node); - break; - - case cmd_health: // dest_node != NULL - case cmd_whois_dc: // dest_node == NULL -- rc = pcmk_controld_api_ping(controld_api, dest_node); -+ rc = pcmk_controld_api_ping(api, dest_node); - need_reply = true; - break; - - case cmd_elect_dc: -- rc = pcmk_controld_api_start_election(controld_api); -+ rc = pcmk_controld_api_start_election(api); - break; - - case cmd_list_nodes: - rc = list_nodes(); - break; - -+ case cmd_pacemakerd_health: -+ rc = pcmk_pacemakerd_api_ping(api, ipc_name); -+ need_reply = true; -+ break; -+ - case cmd_none: // not actually possible here - break; - } --- -1.8.3.1 - - -From 6ce5bb0d6fd30a204468ea245209d34f2682d7c9 Mon Sep 17 00:00:00 2001 -From: Klaus Wenninger -Date: Tue, 21 Jul 2020 18:12:53 +0200 -Subject: [PATCH 5/6] Fix: pacemakerd: interworking with sbd not using - pacemakerd-api - ---- - daemons/pacemakerd/pacemakerd.c | 8 +++++++- - include/crm/common/options_internal.h | 1 + - lib/common/watchdog.c | 15 +++++++++++++++ - 3 files changed, 23 insertions(+), 1 deletion(-) - -diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c -index ccfae66..e91982a 100644 ---- a/daemons/pacemakerd/pacemakerd.c -+++ b/daemons/pacemakerd/pacemakerd.c -@@ -454,6 +454,7 @@ pcmk_shutdown_worker(gpointer user_data) - crm_notice("Shutdown complete"); - pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE; - if (!fatal_error && running_with_sbd && -+ pcmk__get_sbd_sync_resource_startup() && - !shutdown_complete_state_reported_client_closed) { - return TRUE; - } -@@ -1248,10 +1249,15 @@ main(int argc, char **argv) - mainloop_add_signal(SIGTERM, pcmk_shutdown); - mainloop_add_signal(SIGINT, pcmk_shutdown); - -- if (running_with_sbd) { -+ if ((running_with_sbd) && pcmk__get_sbd_sync_resource_startup()) { - pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING; - startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL); - } else { -+ if (running_with_sbd) { -+ crm_warn("Enabling SBD_SYNC_RESOURCE_STARTUP would (if supported " -+ "by your sbd version) improve reliability of " -+ "interworking between SBD & pacemaker."); -+ } - pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS; - init_children_processes(NULL); - } -diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h -index db54da4..d0429c9 100644 ---- a/include/crm/common/options_internal.h -+++ b/include/crm/common/options_internal.h -@@ -111,6 +111,7 @@ bool pcmk__valid_utilization(const char *value); - - // from watchdog.c - long pcmk__get_sbd_timeout(void); -+bool pcmk__get_sbd_sync_resource_startup(void); - long pcmk__auto_watchdog_timeout(void); - bool pcmk__valid_sbd_timeout(const char *value); - -diff --git a/lib/common/watchdog.c b/lib/common/watchdog.c -index 9d8896b..8838be6 100644 ---- a/lib/common/watchdog.c -+++ b/lib/common/watchdog.c -@@ -227,6 +227,21 @@ pcmk__get_sbd_timeout(void) - return sbd_timeout; - } - -+bool -+pcmk__get_sbd_sync_resource_startup(void) -+{ -+ static bool sync_resource_startup = false; -+ static bool checked_sync_resource_startup = false; -+ -+ if (!checked_sync_resource_startup) { -+ sync_resource_startup = -+ crm_is_true(getenv("SBD_SYNC_RESOURCE_STARTUP")); -+ checked_sync_resource_startup = true; -+ } -+ -+ return sync_resource_startup; -+} -+ - long - pcmk__auto_watchdog_timeout() - { --- -1.8.3.1 - - -From 567cb6ec6f317af9e973321633950ef26f43c486 Mon Sep 17 00:00:00 2001 -From: Klaus Wenninger -Date: Thu, 23 Jul 2020 23:00:23 +0200 -Subject: [PATCH 6/6] Fix: pacemakerd: improve logging when synced with SBD - ---- - daemons/pacemakerd/pacemakerd.c | 8 +++++++- - 1 file changed, 7 insertions(+), 1 deletion(-) - -diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c -index e91982a..c888b73 100644 ---- a/daemons/pacemakerd/pacemakerd.c -+++ b/daemons/pacemakerd/pacemakerd.c -@@ -456,6 +456,7 @@ pcmk_shutdown_worker(gpointer user_data) - if (!fatal_error && running_with_sbd && - pcmk__get_sbd_sync_resource_startup() && - !shutdown_complete_state_reported_client_closed) { -+ crm_notice("Waiting for SBD to pick up shutdown-complete-state."); - return TRUE; - } - -@@ -546,10 +547,14 @@ pcmk_handle_ping_request(pcmk__client_t *c, xmlNode *msg, uint32_t id) - if (crm_str_eq(pacemakerd_state, - XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE, - TRUE)) { -+ if (pcmk__get_sbd_sync_resource_startup()) { -+ crm_notice("Shutdown-complete-state passed to SBD."); -+ } - shutdown_complete_state_reported_to = c->pid; - } else if (crm_str_eq(pacemakerd_state, - XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, - TRUE)) { -+ crm_notice("Received startup-trigger from SBD."); - pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS; - mainloop_set_trigger(startup_trigger); - } -@@ -1250,12 +1255,13 @@ main(int argc, char **argv) - mainloop_add_signal(SIGINT, pcmk_shutdown); - - if ((running_with_sbd) && pcmk__get_sbd_sync_resource_startup()) { -+ crm_notice("Waiting for startup-trigger from SBD."); - pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING; - startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL); - } else { - if (running_with_sbd) { - crm_warn("Enabling SBD_SYNC_RESOURCE_STARTUP would (if supported " -- "by your sbd version) improve reliability of " -+ "by your SBD version) improve reliability of " - "interworking between SBD & pacemaker."); - } - pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS; --- -1.8.3.1 - diff --git a/SOURCES/015-cibsecret.patch b/SOURCES/015-cibsecret.patch deleted file mode 100644 index d64cc65..0000000 --- a/SOURCES/015-cibsecret.patch +++ /dev/null @@ -1,123 +0,0 @@ -From 2e7a40570d6b21534ec0215ac5ebc174796cf17c Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 20 Aug 2020 10:02:20 -0500 -Subject: [PATCH 1/2] Refactor: tools: rename function in cibsecret to be more - clear - -It led me to initially misdiagnose a problem. ---- - tools/cibsecret.in | 26 +++++++++++++------------- - 1 file changed, 13 insertions(+), 13 deletions(-) - -diff --git a/tools/cibsecret.in b/tools/cibsecret.in -index 9b74ba3..dabbfc0 100644 ---- a/tools/cibsecret.in -+++ b/tools/cibsecret.in -@@ -162,28 +162,28 @@ check_env() { - } - - # This must be called (and return success) before calling $rsh or $rcp_to_from --get_live_nodes() { -- # Get a list of all cluster nodes -+get_live_peers() { -+ # Get a list of all other cluster nodes - GLN_ALL_NODES="$(crm_node -l | awk '{print $2}' | grep -v "$(uname -n)")" - - # Make a list of those that respond to pings - if [ "$(id -u)" = "0" ] && which fping >/dev/null 2>&1; then -- LIVE_NODES=$(fping -a $GLN_ALL_NODES 2>/dev/null) -+ LIVE_NODES=$(fping -a $GLP_ALL_PEERS 2>/dev/null) - else - LIVE_NODES="" -- for GLN_NODE in $GLN_ALL_NODES; do \ -- ping -c 2 -q "$GLN_NODE" >/dev/null 2>&1 && -- LIVE_NODES="$LIVE_NODES $GLN_NODE" -+ for GLP_NODE in $GLP_ALL_PEERS; do \ -+ ping -c 2 -q "$GLP_NODE" >/dev/null 2>&1 && -+ LIVE_NODES="$LIVE_NODES $GLP_NODE" - done - fi - - # Warn the user about any that didn't respond to pings -- GLN_DOWN="$( (for GLN_NODE in $LIVE_NODES $GLN_ALL_NODES; do echo "$GLN_NODE"; done) | sort | uniq -u)" -- if [ "$(echo "$GLN_DOWN" | wc -w)" = "1" ]; then -- warn "node $GLN_DOWN is down" -+ GLP_DOWN="$( (for GLP_NODE in $LIVE_NODES $GLP_ALL_PEERS; do echo "$GLP_NODE"; done) | sort | uniq -u)" -+ if [ "$(echo "$GLP_DOWN" | wc -w)" = "1" ]; then -+ warn "node $GLP_DOWN is down" - warn "you'll need to update it using \"$PROG sync\" later" -- elif [ -n "$GLN_DOWN" ]; then -- warn "nodes $(echo "$GLN_DOWN" | tr '\n' ' ')are down" -+ elif [ -n "$GLP_DOWN" ]; then -+ warn "nodes $(echo "$GLP_DOWN" | tr '\n' ' ')are down" - warn "you'll need to update them using \"$PROG sync\" later" - fi - -@@ -235,7 +235,7 @@ scp_fun() { - # TODO: this procedure should be replaced with csync2 - # provided that csync2 has already been configured - sync_files() { -- get_live_nodes || return -+ get_live_peers || return - info "syncing $LRM_CIBSECRETS to $(echo "$LIVE_NODES" | tr '\n' ' ') ..." - $rsh rm -rf "$LRM_CIBSECRETS" && - $rsh mkdir -p "$(dirname "$LRM_CIBSECRETS")" && -@@ -244,7 +244,7 @@ sync_files() { - - sync_one() { - SO_FILE="$1" -- get_live_nodes || return -+ get_live_peers || return - info "syncing $SO_FILE to $(echo "$LIVE_NODES" | tr '\n' ' ') ..." - $rsh mkdir -p "$(dirname "$SO_FILE")" && - if [ -f "$SO_FILE" ]; then --- -1.8.3.1 - - -From 9c1517e6a681f35d62b4714e854b258c17ab5e59 Mon Sep 17 00:00:00 2001 -From: Ken Gaillot -Date: Thu, 20 Aug 2020 10:03:23 -0500 -Subject: [PATCH 2/2] Fix: tools: properly detect local node name - -cibsecret had two serious problems when generating a list of other nodes to -sync secrets to: - -* It used `uname -n` to remove the local node from the list. If the local node - name is different from its uname, this could cause local secrets to be - removed from the local node rather than synced to other nodes. - -* It removed not just the local node name, but any node name that contained - the local node name as a substring (e.g. "node1" and "node10"). This could - cause secrets to not be synced to such nodes. - -Now, use `crm_node -n` to determine the local node name, check crm_node for -errors to get better error messages, and remove only the node name that matches -the local node name in its entirety. ---- - tools/cibsecret.in | 8 +++++++- - 1 file changed, 7 insertions(+), 1 deletion(-) - -diff --git a/tools/cibsecret.in b/tools/cibsecret.in -index dabbfc0..568833c 100644 ---- a/tools/cibsecret.in -+++ b/tools/cibsecret.in -@@ -163,8 +163,14 @@ check_env() { - - # This must be called (and return success) before calling $rsh or $rcp_to_from - get_live_peers() { -+ # Get local node name -+ GLP_LOCAL_NODE="$(crm_node -n)" -+ [ $? -eq 0 ] || fatal "couldn't get local node name" -+ - # Get a list of all other cluster nodes -- GLN_ALL_NODES="$(crm_node -l | awk '{print $2}' | grep -v "$(uname -n)")" -+ GLP_ALL_PEERS="$(crm_node -l)" -+ [ $? -eq 0 ] || fatal "couldn't determine cluster nodes" -+ GLP_ALL_PEERS="$(echo "$GLP_ALL_PEERS" | awk '{print $2}' | grep -v "^${GLP_LOCAL_NODE}$")" - - # Make a list of those that respond to pings - if [ "$(id -u)" = "0" ] && which fping >/dev/null 2>&1; then --- -1.8.3.1 - diff --git a/SPECS/pacemaker.spec b/SPECS/pacemaker.spec index b33bb5f..56139d3 100644 --- a/SPECS/pacemaker.spec +++ b/SPECS/pacemaker.spec @@ -22,11 +22,11 @@ ## Upstream pacemaker version, and its package version (specversion ## can be incremented to build packages reliably considered "newer" ## than previously built packages with the same pcmkversion) -%global pcmkversion 2.0.4 -%global specversion 6 +%global pcmkversion 2.0.5 +%global specversion 2 ## Upstream commit (full commit ID, abbreviated commit ID, or tag) to build -%global commit 2deceaa3ae1fbadd844f5c5b47fd33129fa2c227 +%global commit 31aa4f551559cea7c4341c5bf3907a4ca5ba3897 ## Since git v2.11, the extent of abbreviation is autoscaled by default ## (used to be constant of 7), so we need to convey it for non-tags, too. %global commit_abbrev 7 @@ -214,7 +214,6 @@ } %endif - # Keep sane profiling data if requested %if %{with profiling} @@ -234,7 +233,7 @@ License: GPLv2+ and LGPLv2+ # initscript is Revised BSD License: GPLv2+ and LGPLv2+ and BSD %endif -Url: http://www.clusterlabs.org +Url: https://www.clusterlabs.org/ Group: System Environment/Daemons # Example: https://codeload.github.com/ClusterLabs/pacemaker/tar.gz/e91769e @@ -249,21 +248,7 @@ Source0: https://codeload.github.com/%{github_owner}/%{name}/tar.gz/%{arch Source1: nagios-agents-metadata-%{nagios_hash}.tar.gz # upstream commits -Patch1: 001-rules.patch -Patch2: 002-demote.patch -Patch3: 003-trace.patch -Patch4: 004-test.patch -Patch5: 005-sysconfig.patch -Patch6: 006-ipc_refactor.patch -Patch7: 007-ipc_model.patch -Patch8: 008-crm_node.patch -Patch9: 009-timeout-log.patch -Patch10: 010-crm_mon.patch -Patch11: 011-cts.patch -Patch12: 012-ipc_fix.patch -Patch13: 013-pacemakerd.patch -Patch14: 014-sbd.patch -Patch15: 015-cibsecret.patch +#Patch1: 001-xxx.patch # downstream-only commits #Patch100: xxx.patch @@ -525,9 +510,6 @@ monitor resources. %build -# Early versions of autotools (e.g. RHEL <= 5) do not support --docdir -export docdir=%{pcmk_docdir} - export systemdsystemunitdir=%{?_unitdir}%{!?_unitdir:no} # RHEL changes pacemaker's concurrent-fencing default to true @@ -961,6 +943,34 @@ exit 0 %license %{nagios_name}-%{nagios_hash}/COPYING %changelog +* Wed Oct 28 2020 Ken Gaillot - 2.0.5-2 +- Rebase on upstream 2.0.5-rc2 release +- Prevent ACL bypass (CVE-2020-25654) +- Resolves: rhbz1885645 +- Resolves: rhbz1889582 + +* Tue Oct 20 2020 Ken Gaillot - 2.0.5-1 +- crm_mon --resource option to filter output by resource +- Avoid filling /dev/shm after frequent corosync errors +- Allow configurable permissions for log files +- Ensure ACL write permission always supersedes read +- Use fence device monitor timeout for starts and probes +- Allow type="integer" in rule expressions +- Avoid error messages when running crm_node inside bundles +- Avoid situation where promotion is not scheduled until next transition +- crm_mon should show more clearly when an entire group is disabled +- Rebase on upstream 2.0.5-rc1 release +- Resolves: rhbz1300597 +- Resolves: rhbz1614166 +- Resolves: rhbz1647136 +- Resolves: rhbz1833173 +- Resolves: rhbz1856015 +- Resolves: rhbz1866573 +- Resolves: rhbz1874391 +- Resolves: rhbz1835717 +- Resolves: rhbz1748139 +- Resolves: rhbz1885645 + * Thu Aug 20 2020 Ken Gaillot - 2.0.4-6 - Fix cibsecret bug when node name is different from hostname - Resolves: rhbz1870873