Blame SOURCES/010-probe-failures.patch

33afe3
From f2e51898735b5e9990464141fc4aea3dd83f5067 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Thu, 4 Nov 2021 14:36:41 -0400
33afe3
Subject: [PATCH 01/21] Refactor: scheduler: Use bool in unpack_rsc_op.
33afe3
33afe3
Previously, we were using bool but TRUE/FALSE.  Instead, use the actual
33afe3
values.
33afe3
---
33afe3
 lib/pengine/unpack.c | 4 ++--
33afe3
 1 file changed, 2 insertions(+), 2 deletions(-)
33afe3
33afe3
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
33afe3
index b1e84110a2..ecc7275e15 100644
33afe3
--- a/lib/pengine/unpack.c
33afe3
+++ b/lib/pengine/unpack.c
33afe3
@@ -3671,7 +3671,7 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
33afe3
     const char *task = NULL;
33afe3
     const char *task_key = NULL;
33afe3
     const char *exit_reason = NULL;
33afe3
-    bool expired = FALSE;
33afe3
+    bool expired = false;
33afe3
     pe_resource_t *parent = rsc;
33afe3
     enum action_fail_response failure_strategy = action_fail_recover;
33afe3
 
33afe3
@@ -3727,7 +3727,7 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
33afe3
 
33afe3
     if ((status != PCMK_EXEC_NOT_INSTALLED)
33afe3
         && check_operation_expiry(rsc, node, rc, xml_op, data_set)) {
33afe3
-        expired = TRUE;
33afe3
+        expired = true;
33afe3
     }
33afe3
 
33afe3
     if (!strcmp(task, CRMD_ACTION_STATUS)) {
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 4c961b8e670d336a368c7fd1535c247e40c6b48e Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Thu, 4 Nov 2021 15:07:01 -0400
33afe3
Subject: [PATCH 02/21] Refactor: scheduler: Add functions for determining if
33afe3
 an op is a probe.
33afe3
33afe3
---
33afe3
 include/crm/common/util.h                     |  3 +
33afe3
 lib/common/operations.c                       | 21 +++++++
33afe3
 lib/common/tests/operations/Makefile.am       |  6 +-
33afe3
 .../tests/operations/pcmk_is_probe_test.c     | 37 +++++++++++++
33afe3
 .../tests/operations/pcmk_xe_is_probe_test.c  | 55 +++++++++++++++++++
33afe3
 lib/pengine/unpack.c                          | 12 ++--
33afe3
 lib/pengine/utils.c                           |  5 +-
33afe3
 7 files changed, 127 insertions(+), 12 deletions(-)
33afe3
 create mode 100644 lib/common/tests/operations/pcmk_is_probe_test.c
33afe3
 create mode 100644 lib/common/tests/operations/pcmk_xe_is_probe_test.c
33afe3
33afe3
diff --git a/include/crm/common/util.h b/include/crm/common/util.h
33afe3
index 2728b64492..fbea6e560c 100644
33afe3
--- a/include/crm/common/util.h
33afe3
+++ b/include/crm/common/util.h
33afe3
@@ -72,6 +72,9 @@ xmlNode *crm_create_op_xml(xmlNode *parent, const char *prefix,
33afe3
                            const char *timeout);
33afe3
 #define CRM_DEFAULT_OP_TIMEOUT_S "20s"
33afe3
 
33afe3
+bool pcmk_is_probe(const char *task, guint interval);
33afe3
+bool pcmk_xe_is_probe(xmlNode *xml_op);
33afe3
+
33afe3
 int compare_version(const char *version1, const char *version2);
33afe3
 
33afe3
 /* coverity[+kill] */
33afe3
diff --git a/lib/common/operations.c b/lib/common/operations.c
33afe3
index 366c189702..978df79082 100644
33afe3
--- a/lib/common/operations.c
33afe3
+++ b/lib/common/operations.c
33afe3
@@ -537,3 +537,24 @@ pcmk__is_fencing_action(const char *action)
33afe3
 {
33afe3
     return pcmk__str_any_of(action, "off", "reboot", "poweroff", NULL);
33afe3
 }
33afe3
+
33afe3
+bool
33afe3
+pcmk_is_probe(const char *task, guint interval)
33afe3
+{
33afe3
+    if (task == NULL) {
33afe3
+        return false;
33afe3
+    }
33afe3
+
33afe3
+    return (interval == 0) && pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_none);
33afe3
+}
33afe3
+
33afe3
+bool
33afe3
+pcmk_xe_is_probe(xmlNode *xml_op)
33afe3
+{
33afe3
+    const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
33afe3
+    const char *interval_ms_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
33afe3
+    int interval_ms;
33afe3
+
33afe3
+    pcmk__scan_min_int(interval_ms_s, &interval_ms, 0);
33afe3
+    return pcmk_is_probe(task, interval_ms);
33afe3
+}
33afe3
diff --git a/lib/common/tests/operations/Makefile.am b/lib/common/tests/operations/Makefile.am
33afe3
index c8814ff0a8..2e3d0b0679 100644
33afe3
--- a/lib/common/tests/operations/Makefile.am
33afe3
+++ b/lib/common/tests/operations/Makefile.am
33afe3
@@ -1,5 +1,5 @@
33afe3
 #
33afe3
-# Copyright 2020 the Pacemaker project contributors
33afe3
+# Copyright 2020-2021 the Pacemaker project contributors
33afe3
 #
33afe3
 # The version control history for this file may have further details.
33afe3
 #
33afe3
@@ -12,6 +12,8 @@ LDADD = $(top_builddir)/lib/common/libcrmcommon.la -lcmocka
33afe3
 include $(top_srcdir)/mk/tap.mk
33afe3
 
33afe3
 # Add "_test" to the end of all test program names to simplify .gitignore.
33afe3
-check_PROGRAMS = parse_op_key_test
33afe3
+check_PROGRAMS = parse_op_key_test \
33afe3
+				 pcmk_is_probe_test \
33afe3
+				 pcmk_xe_is_probe_test
33afe3
 
33afe3
 TESTS = $(check_PROGRAMS)
33afe3
diff --git a/lib/common/tests/operations/pcmk_is_probe_test.c b/lib/common/tests/operations/pcmk_is_probe_test.c
33afe3
new file mode 100644
33afe3
index 0000000000..9b449f1a70
33afe3
--- /dev/null
33afe3
+++ b/lib/common/tests/operations/pcmk_is_probe_test.c
33afe3
@@ -0,0 +1,37 @@
33afe3
+/*
33afe3
+ * Copyright 2021 the Pacemaker project contributors
33afe3
+ *
33afe3
+ * The version control history for this file may have further details.
33afe3
+ *
33afe3
+ * This source code is licensed under the GNU Lesser General Public License
33afe3
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
33afe3
+ */
33afe3
+
33afe3
+#include <crm_internal.h>
33afe3
+
33afe3
+#include <stdarg.h>
33afe3
+#include <stddef.h>
33afe3
+#include <stdint.h>
33afe3
+#include <stdlib.h>
33afe3
+#include <setjmp.h>
33afe3
+#include <cmocka.h>
33afe3
+
33afe3
+static void
33afe3
+is_probe_test(void **state)
33afe3
+{
33afe3
+    assert_false(pcmk_is_probe(NULL, 0));
33afe3
+    assert_false(pcmk_is_probe("", 0));
33afe3
+    assert_false(pcmk_is_probe("blahblah", 0));
33afe3
+    assert_false(pcmk_is_probe("monitor", 1));
33afe3
+    assert_true(pcmk_is_probe("monitor", 0));
33afe3
+}
33afe3
+
33afe3
+int main(int argc, char **argv)
33afe3
+{
33afe3
+    const struct CMUnitTest tests[] = {
33afe3
+        cmocka_unit_test(is_probe_test),
33afe3
+    };
33afe3
+
33afe3
+    cmocka_set_message_output(CM_OUTPUT_TAP);
33afe3
+    return cmocka_run_group_tests(tests, NULL, NULL);
33afe3
+}
33afe3
diff --git a/lib/common/tests/operations/pcmk_xe_is_probe_test.c b/lib/common/tests/operations/pcmk_xe_is_probe_test.c
33afe3
new file mode 100644
33afe3
index 0000000000..0283d1c145
33afe3
--- /dev/null
33afe3
+++ b/lib/common/tests/operations/pcmk_xe_is_probe_test.c
33afe3
@@ -0,0 +1,55 @@
33afe3
+/*
33afe3
+ * Copyright 2021 the Pacemaker project contributors
33afe3
+ *
33afe3
+ * The version control history for this file may have further details.
33afe3
+ *
33afe3
+ * This source code is licensed under the GNU Lesser General Public License
33afe3
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
33afe3
+ */
33afe3
+
33afe3
+#include <crm_internal.h>
33afe3
+
33afe3
+#include <stdarg.h>
33afe3
+#include <stddef.h>
33afe3
+#include <stdint.h>
33afe3
+#include <stdlib.h>
33afe3
+#include <setjmp.h>
33afe3
+#include <cmocka.h>
33afe3
+
33afe3
+static void
33afe3
+op_is_probe_test(void **state)
33afe3
+{
33afe3
+    xmlNode *node = NULL;
33afe3
+
33afe3
+    assert_false(pcmk_xe_is_probe(NULL));
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op/>");
33afe3
+    assert_false(pcmk_xe_is_probe(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation_key=\"blah\" interval=\"30s\"/>");
33afe3
+    assert_false(pcmk_xe_is_probe(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"30s\"/>");
33afe3
+    assert_false(pcmk_xe_is_probe(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"start\" interval=\"0\"/>");
33afe3
+    assert_false(pcmk_xe_is_probe(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\"/>");
33afe3
+    assert_true(pcmk_xe_is_probe(node));
33afe3
+    free_xml(node);
33afe3
+}
33afe3
+
33afe3
+int main(int argc, char **argv)
33afe3
+{
33afe3
+    const struct CMUnitTest tests[] = {
33afe3
+        cmocka_unit_test(op_is_probe_test),
33afe3
+    };
33afe3
+
33afe3
+    cmocka_set_message_output(CM_OUTPUT_TAP);
33afe3
+    return cmocka_run_group_tests(tests, NULL, NULL);
33afe3
+}
33afe3
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
33afe3
index ecc7275e15..7c0c66e696 100644
33afe3
--- a/lib/pengine/unpack.c
33afe3
+++ b/lib/pengine/unpack.c
33afe3
@@ -83,7 +83,6 @@ is_dangling_guest_node(pe_node_t *node)
33afe3
     return FALSE;
33afe3
 }
33afe3
 
33afe3
-
33afe3
 /*!
33afe3
  * \brief Schedule a fence action for a node
33afe3
  *
33afe3
@@ -2984,7 +2983,6 @@ static void
33afe3
 unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * xml_op, xmlNode ** last_failure,
33afe3
                       enum action_fail_response * on_fail, pe_working_set_t * data_set)
33afe3
 {
33afe3
-    guint interval_ms = 0;
33afe3
     bool is_probe = false;
33afe3
     pe_action_t *action = NULL;
33afe3
 
33afe3
@@ -2998,10 +2996,7 @@ unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * x
33afe3
 
33afe3
     *last_failure = xml_op;
33afe3
 
33afe3
-    crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
33afe3
-    if ((interval_ms == 0) && !strcmp(task, CRMD_ACTION_STATUS)) {
33afe3
-        is_probe = true;
33afe3
-    }
33afe3
+    is_probe = pcmk_xe_is_probe(xml_op);
33afe3
 
33afe3
     if (exit_reason == NULL) {
33afe3
         exit_reason = "";
33afe3
@@ -3163,8 +3158,9 @@ determine_op_status(
33afe3
     }
33afe3
 
33afe3
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
33afe3
-    if ((interval_ms == 0) && !strcmp(task, CRMD_ACTION_STATUS)) {
33afe3
-        is_probe = true;
33afe3
+    is_probe = pcmk_xe_is_probe(xml_op);
33afe3
+
33afe3
+    if (is_probe) {
33afe3
         task = "probe";
33afe3
     }
33afe3
 
33afe3
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
33afe3
index c5eda3898e..07753e173a 100644
33afe3
--- a/lib/pengine/utils.c
33afe3
+++ b/lib/pengine/utils.c
33afe3
@@ -1066,8 +1066,7 @@ unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * contai
33afe3
 {
33afe3
     int timeout_ms = 0;
33afe3
     const char *value = NULL;
33afe3
-    bool is_probe = pcmk__str_eq(action->task, RSC_STATUS, pcmk__str_casei)
33afe3
-                    && (interval_ms == 0);
33afe3
+    bool is_probe = false;
33afe3
 #if ENABLE_VERSIONED_ATTRS
33afe3
     pe_rsc_action_details_t *rsc_details = NULL;
33afe3
 #endif
33afe3
@@ -1094,6 +1093,8 @@ unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * contai
33afe3
 
33afe3
     CRM_CHECK(action && action->rsc, return);
33afe3
 
33afe3
+    is_probe = pcmk_is_probe(action->task, interval_ms);
33afe3
+
33afe3
     // Cluster-wide <op_defaults> <meta_attributes>
33afe3
     pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS, &rule_data,
33afe3
                                action->meta, NULL, FALSE, data_set);
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 09f32df97ab5064a15ba5a1fb3970d5c64ee7b30 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Fri, 19 Nov 2021 14:47:22 -0500
33afe3
Subject: [PATCH 03/21] Refactor: scheduler: Move setting interval_ms in
33afe3
 determine_op_status.
33afe3
33afe3
This can now happen in the only place it's being used.
33afe3
---
33afe3
 lib/pengine/unpack.c | 9 ++++++---
33afe3
 1 file changed, 6 insertions(+), 3 deletions(-)
33afe3
33afe3
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
33afe3
index 7c0c66e696..b9986d2462 100644
33afe3
--- a/lib/pengine/unpack.c
33afe3
+++ b/lib/pengine/unpack.c
33afe3
@@ -3142,7 +3142,6 @@ static int
33afe3
 determine_op_status(
33afe3
     pe_resource_t *rsc, int rc, int target_rc, pe_node_t * node, xmlNode * xml_op, enum action_fail_response * on_fail, pe_working_set_t * data_set) 
33afe3
 {
33afe3
-    guint interval_ms = 0;
33afe3
     bool is_probe = false;
33afe3
     int result = PCMK_EXEC_DONE;
33afe3
     const char *key = get_op_key(xml_op);
33afe3
@@ -3157,7 +3156,6 @@ determine_op_status(
33afe3
         exit_reason = "";
33afe3
     }
33afe3
 
33afe3
-    crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
33afe3
     is_probe = pcmk_xe_is_probe(xml_op);
33afe3
 
33afe3
     if (is_probe) {
33afe3
@@ -3230,12 +3228,17 @@ determine_op_status(
33afe3
             result = PCMK_EXEC_ERROR_FATAL;
33afe3
             break;
33afe3
 
33afe3
-        case PCMK_OCF_UNIMPLEMENT_FEATURE:
33afe3
+        case PCMK_OCF_UNIMPLEMENT_FEATURE: {
33afe3
+            guint interval_ms = 0;
33afe3
+            crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
33afe3
+
33afe3
             if (interval_ms > 0) {
33afe3
                 result = PCMK_EXEC_NOT_SUPPORTED;
33afe3
                 break;
33afe3
             }
33afe3
             // fall through
33afe3
+        }
33afe3
+
33afe3
         case PCMK_OCF_NOT_INSTALLED:
33afe3
         case PCMK_OCF_INVALID_PARAM:
33afe3
         case PCMK_OCF_INSUFFICIENT_PRIV:
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 6c8f47453afd6c100fddc45187faff17e15f7bfe Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Fri, 19 Nov 2021 14:57:57 -0500
33afe3
Subject: [PATCH 04/21] Refactor: scheduler: Add pcmk_xe_mask_failed_probe.
33afe3
33afe3
Given an xmlNodePtr for a resource operation, this function will
33afe3
determine whether it is a failed probe operation that should not be
33afe3
displayed in crm_mon (or other places, I suppose) or not.
33afe3
---
33afe3
 include/crm/common/util.h                     |   1 +
33afe3
 lib/common/operations.c                       |  17 ++
33afe3
 lib/common/tests/operations/Makefile.am       |   3 +-
33afe3
 .../pcmk_xe_mask_probe_failure_test.c         | 162 ++++++++++++++++++
33afe3
 4 files changed, 182 insertions(+), 1 deletion(-)
33afe3
 create mode 100644 lib/common/tests/operations/pcmk_xe_mask_probe_failure_test.c
33afe3
33afe3
diff --git a/include/crm/common/util.h b/include/crm/common/util.h
33afe3
index fbea6e560c..784069ba1b 100644
33afe3
--- a/include/crm/common/util.h
33afe3
+++ b/include/crm/common/util.h
33afe3
@@ -74,6 +74,7 @@ xmlNode *crm_create_op_xml(xmlNode *parent, const char *prefix,
33afe3
 
33afe3
 bool pcmk_is_probe(const char *task, guint interval);
33afe3
 bool pcmk_xe_is_probe(xmlNode *xml_op);
33afe3
+bool pcmk_xe_mask_probe_failure(xmlNode *xml_op);
33afe3
 
33afe3
 int compare_version(const char *version1, const char *version2);
33afe3
 
33afe3
diff --git a/lib/common/operations.c b/lib/common/operations.c
33afe3
index 978df79082..54482b8863 100644
33afe3
--- a/lib/common/operations.c
33afe3
+++ b/lib/common/operations.c
33afe3
@@ -558,3 +558,20 @@ pcmk_xe_is_probe(xmlNode *xml_op)
33afe3
     pcmk__scan_min_int(interval_ms_s, &interval_ms, 0);
33afe3
     return pcmk_is_probe(task, interval_ms);
33afe3
 }
33afe3
+
33afe3
+bool
33afe3
+pcmk_xe_mask_probe_failure(xmlNode *xml_op)
33afe3
+{
33afe3
+    int status = PCMK_EXEC_UNKNOWN;
33afe3
+    int rc = PCMK_OCF_OK;
33afe3
+
33afe3
+    if (!pcmk_xe_is_probe(xml_op)) {
33afe3
+        return false;
33afe3
+    }
33afe3
+
33afe3
+    crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, &status);
33afe3
+    crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &rc);
33afe3
+
33afe3
+    return rc == PCMK_OCF_NOT_INSTALLED || rc == PCMK_OCF_INVALID_PARAM ||
33afe3
+           status == PCMK_EXEC_NOT_INSTALLED;
33afe3
+}
33afe3
diff --git a/lib/common/tests/operations/Makefile.am b/lib/common/tests/operations/Makefile.am
33afe3
index 2e3d0b0679..457c5f7c7a 100644
33afe3
--- a/lib/common/tests/operations/Makefile.am
33afe3
+++ b/lib/common/tests/operations/Makefile.am
33afe3
@@ -14,6 +14,7 @@ include $(top_srcdir)/mk/tap.mk
33afe3
 # Add "_test" to the end of all test program names to simplify .gitignore.
33afe3
 check_PROGRAMS = parse_op_key_test \
33afe3
 				 pcmk_is_probe_test \
33afe3
-				 pcmk_xe_is_probe_test
33afe3
+				 pcmk_xe_is_probe_test \
33afe3
+				 pcmk_xe_mask_probe_failure_test
33afe3
 
33afe3
 TESTS = $(check_PROGRAMS)
33afe3
diff --git a/lib/common/tests/operations/pcmk_xe_mask_probe_failure_test.c b/lib/common/tests/operations/pcmk_xe_mask_probe_failure_test.c
33afe3
new file mode 100644
33afe3
index 0000000000..a13f6d98f4
33afe3
--- /dev/null
33afe3
+++ b/lib/common/tests/operations/pcmk_xe_mask_probe_failure_test.c
33afe3
@@ -0,0 +1,162 @@
33afe3
+/*
33afe3
+ * Copyright 2021 the Pacemaker project contributors
33afe3
+ *
33afe3
+ * The version control history for this file may have further details.
33afe3
+ *
33afe3
+ * This source code is licensed under the GNU Lesser General Public License
33afe3
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
33afe3
+ */
33afe3
+
33afe3
+#include <crm_internal.h>
33afe3
+
33afe3
+#include <stdarg.h>
33afe3
+#include <stddef.h>
33afe3
+#include <stdint.h>
33afe3
+#include <stdlib.h>
33afe3
+#include <setjmp.h>
33afe3
+#include <cmocka.h>
33afe3
+
33afe3
+static void
33afe3
+op_is_not_probe_test(void **state) {
33afe3
+    xmlNode *node = NULL;
33afe3
+
33afe3
+    /* Not worth testing this thoroughly since it's just a duplicate of whether
33afe3
+     * pcmk_op_is_probe works or not.
33afe3
+     */
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"start\" interval=\"0\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+}
33afe3
+
33afe3
+static void
33afe3
+op_does_not_have_right_values_test(void **state) {
33afe3
+    xmlNode *node = NULL;
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+}
33afe3
+
33afe3
+static void
33afe3
+check_values_test(void **state) {
33afe3
+    xmlNode *node = NULL;
33afe3
+
33afe3
+    /* PCMK_EXEC_NOT_SUPPORTED */
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"3\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"3\"/>");
33afe3
+    assert_true(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    /* PCMK_EXEC_DONE */
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"0\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"2\" op-status=\"0\"/>");
33afe3
+    assert_true(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"0\"/>");
33afe3
+    assert_true(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"6\" op-status=\"0\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"7\" op-status=\"0\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    /* PCMK_EXEC_NOT_INSTALLED */
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"7\"/>");
33afe3
+    assert_true(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"7\"/>");
33afe3
+    assert_true(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    /* PCMK_EXEC_ERROR */
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"4\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"2\" op-status=\"4\"/>");
33afe3
+    assert_true(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"4\"/>");
33afe3
+    assert_true(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"6\" op-status=\"4\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"7\" op-status=\"4\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    /* PCMK_EXEC_ERROR_HARD */
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"5\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"2\" op-status=\"5\"/>");
33afe3
+    assert_true(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"5\"/>");
33afe3
+    assert_true(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"6\" op-status=\"5\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"7\" op-status=\"5\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    /* PCMK_EXEC_ERROR_FATAL */
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"6\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"2\" op-status=\"6\"/>");
33afe3
+    assert_true(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"6\"/>");
33afe3
+    assert_true(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"6\" op-status=\"6\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+
33afe3
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"7\" op-status=\"6\"/>");
33afe3
+    assert_false(pcmk_xe_mask_probe_failure(node));
33afe3
+    free_xml(node);
33afe3
+}
33afe3
+
33afe3
+int main(int argc, char **argv)
33afe3
+{
33afe3
+    const struct CMUnitTest tests[] = {
33afe3
+        cmocka_unit_test(op_is_not_probe_test),
33afe3
+        cmocka_unit_test(op_does_not_have_right_values_test),
33afe3
+        cmocka_unit_test(check_values_test),
33afe3
+    };
33afe3
+
33afe3
+    cmocka_set_message_output(CM_OUTPUT_TAP);
33afe3
+    return cmocka_run_group_tests(tests, NULL, NULL);
33afe3
+}
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From c9ce1aaf93cd20bb01e80102dda0ffffb07e6472 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Wed, 1 Dec 2021 14:26:31 -0500
33afe3
Subject: [PATCH 05/21] Refactor: scheduler: Combine op status and rc remapping
33afe3
 into one function.
33afe3
33afe3
Well, not quite.  Doing the remapping is complicated enough to where it
33afe3
makes sense to have them in separate functions.  However, they can both
33afe3
be called from a single new function that takes the place of the
33afe3
previous two calls in unpack_rsc_op.
33afe3
---
33afe3
 lib/pengine/unpack.c | 157 ++++++++++++++++++++-----------------------
33afe3
 1 file changed, 72 insertions(+), 85 deletions(-)
33afe3
33afe3
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
33afe3
index b9986d2462..b659f319fb 100644
33afe3
--- a/lib/pengine/unpack.c
33afe3
+++ b/lib/pengine/unpack.c
33afe3
@@ -3121,36 +3121,68 @@ unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * x
33afe3
 
33afe3
 /*!
33afe3
  * \internal
33afe3
- * \brief Remap operation status based on action result
33afe3
+ * \brief Remap informational monitor results and operation status
33afe3
  *
33afe3
- * Given an action result, determine an appropriate operation status for the
33afe3
- * purposes of responding to the action (the status provided by the executor is
33afe3
- * not directly usable since the executor does not know what was expected).
33afe3
+ * For the monitor results, certain OCF codes are for providing extended information
33afe3
+ * to the user about services that aren't yet failed but not entirely healthy either.
33afe3
+ * These must be treated as the "normal" result by Pacemaker.
33afe3
+ *
33afe3
+ * For operation status, the action result can be used to determine an appropriate
33afe3
+ * status for the purposes of responding to the action.  The status provided by the
33afe3
+ * executor is not directly usable since the executor does not know what was expected.
33afe3
  *
33afe3
+ * \param[in]     xml_op     Operation history entry XML from CIB status
33afe3
  * \param[in,out] rsc        Resource that operation history entry is for
33afe3
- * \param[in]     rc         Actual return code of operation
33afe3
- * \param[in]     target_rc  Expected return code of operation
33afe3
  * \param[in]     node       Node where operation was executed
33afe3
- * \param[in]     xml_op     Operation history entry XML from CIB status
33afe3
- * \param[in,out] on_fail    What should be done about the result
33afe3
  * \param[in]     data_set   Current cluster working set
33afe3
+ * \param[in,out] on_fail    What should be done about the result
33afe3
+ * \param[in]     target_rc  Expected return code of operation
33afe3
+ * \param[in,out] rc         Actual return code of operation
33afe3
+ * \param[in,out] status     Operation execution status
33afe3
+ *
33afe3
+ * \note If the result is remapped and the node is not shutting down or failed,
33afe3
+ *       the operation will be recorded in the data set's list of failed operations
33afe3
+ *       to highlight it for the user.
33afe3
  *
33afe3
- * \return Operation status based on return code and action info
33afe3
  * \note This may update the resource's current and next role.
33afe3
  */
33afe3
-static int
33afe3
-determine_op_status(
33afe3
-    pe_resource_t *rsc, int rc, int target_rc, pe_node_t * node, xmlNode * xml_op, enum action_fail_response * on_fail, pe_working_set_t * data_set) 
33afe3
-{
33afe3
+static void
33afe3
+remap_operation(xmlNode *xml_op, pe_resource_t *rsc, pe_node_t *node,
33afe3
+                pe_working_set_t *data_set, enum action_fail_response *on_fail,
33afe3
+                int target_rc, int *rc, int *status) {
33afe3
     bool is_probe = false;
33afe3
-    int result = PCMK_EXEC_DONE;
33afe3
-    const char *key = get_op_key(xml_op);
33afe3
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
33afe3
+    const char *key = get_op_key(xml_op);
33afe3
     const char *exit_reason = crm_element_value(xml_op,
33afe3
                                                 XML_LRM_ATTR_EXIT_REASON);
33afe3
 
33afe3
+    if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_none)) {
33afe3
+        int remapped_rc = pcmk__effective_rc(*rc);
33afe3
+
33afe3
+        if (*rc != remapped_rc) {
33afe3
+            crm_trace("Remapping monitor result %d to %d", *rc, remapped_rc);
33afe3
+            if (!node->details->shutdown || node->details->online) {
33afe3
+                record_failed_op(xml_op, node, rsc, data_set);
33afe3
+            }
33afe3
+
33afe3
+            *rc = remapped_rc;
33afe3
+        }
33afe3
+    }
33afe3
+
33afe3
+    /* If the executor reported an operation status of anything but done or
33afe3
+     * error, consider that final. But for done or error, we know better whether
33afe3
+     * it should be treated as a failure or not, because we know the expected
33afe3
+     * result.
33afe3
+     */
33afe3
+    if (*status != PCMK_EXEC_DONE && *status != PCMK_EXEC_ERROR) {
33afe3
+        return;
33afe3
+    }
33afe3
+
33afe3
     CRM_ASSERT(rsc);
33afe3
-    CRM_CHECK(task != NULL, return PCMK_EXEC_ERROR);
33afe3
+    CRM_CHECK(task != NULL,
33afe3
+              *status = PCMK_EXEC_ERROR; return);
33afe3
+
33afe3
+    *status = PCMK_EXEC_DONE;
33afe3
 
33afe3
     if (exit_reason == NULL) {
33afe3
         exit_reason = "";
33afe3
@@ -3171,23 +3203,23 @@ determine_op_status(
33afe3
          * those versions or processing of saved CIB files from those versions,
33afe3
          * so we do not need to care much about this case.
33afe3
          */
33afe3
-        result = PCMK_EXEC_ERROR;
33afe3
+        *status = PCMK_EXEC_ERROR;
33afe3
         crm_warn("Expected result not found for %s on %s (corrupt or obsolete CIB?)",
33afe3
                  key, node->details->uname);
33afe3
 
33afe3
-    } else if (target_rc != rc) {
33afe3
-        result = PCMK_EXEC_ERROR;
33afe3
+    } else if (target_rc != *rc) {
33afe3
+        *status = PCMK_EXEC_ERROR;
33afe3
         pe_rsc_debug(rsc, "%s on %s: expected %d (%s), got %d (%s%s%s)",
33afe3
                      key, node->details->uname,
33afe3
                      target_rc, services_ocf_exitcode_str(target_rc),
33afe3
-                     rc, services_ocf_exitcode_str(rc),
33afe3
+                     *rc, services_ocf_exitcode_str(*rc),
33afe3
                      (*exit_reason? ": " : ""), exit_reason);
33afe3
     }
33afe3
 
33afe3
-    switch (rc) {
33afe3
+    switch (*rc) {
33afe3
         case PCMK_OCF_OK:
33afe3
             if (is_probe && (target_rc == PCMK_OCF_NOT_RUNNING)) {
33afe3
-                result = PCMK_EXEC_DONE;
33afe3
+                *status = PCMK_EXEC_DONE;
33afe3
                 pe_rsc_info(rsc, "Probe found %s active on %s at %s",
33afe3
                             rsc->id, node->details->uname,
33afe3
                             last_change_str(xml_op));
33afe3
@@ -3195,10 +3227,10 @@ determine_op_status(
33afe3
             break;
33afe3
 
33afe3
         case PCMK_OCF_NOT_RUNNING:
33afe3
-            if (is_probe || (target_rc == rc)
33afe3
+            if (is_probe || (target_rc == *rc)
33afe3
                 || !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
33afe3
 
33afe3
-                result = PCMK_EXEC_DONE;
33afe3
+                *status = PCMK_EXEC_DONE;
33afe3
                 rsc->role = RSC_ROLE_STOPPED;
33afe3
 
33afe3
                 /* clear any previous failure actions */
33afe3
@@ -3208,8 +3240,8 @@ determine_op_status(
33afe3
             break;
33afe3
 
33afe3
         case PCMK_OCF_RUNNING_PROMOTED:
33afe3
-            if (is_probe && (rc != target_rc)) {
33afe3
-                result = PCMK_EXEC_DONE;
33afe3
+            if (is_probe && (*rc != target_rc)) {
33afe3
+                *status = PCMK_EXEC_DONE;
33afe3
                 pe_rsc_info(rsc,
33afe3
                             "Probe found %s active and promoted on %s at %s",
33afe3
                             rsc->id, node->details->uname,
33afe3
@@ -3221,11 +3253,11 @@ determine_op_status(
33afe3
         case PCMK_OCF_DEGRADED_PROMOTED:
33afe3
         case PCMK_OCF_FAILED_PROMOTED:
33afe3
             rsc->role = RSC_ROLE_PROMOTED;
33afe3
-            result = PCMK_EXEC_ERROR;
33afe3
+            *status = PCMK_EXEC_ERROR;
33afe3
             break;
33afe3
 
33afe3
         case PCMK_OCF_NOT_CONFIGURED:
33afe3
-            result = PCMK_EXEC_ERROR_FATAL;
33afe3
+            *status = PCMK_EXEC_ERROR_FATAL;
33afe3
             break;
33afe3
 
33afe3
         case PCMK_OCF_UNIMPLEMENT_FEATURE: {
33afe3
@@ -3233,7 +3265,7 @@ determine_op_status(
33afe3
             crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
33afe3
 
33afe3
             if (interval_ms > 0) {
33afe3
-                result = PCMK_EXEC_NOT_SUPPORTED;
33afe3
+                *status = PCMK_EXEC_NOT_SUPPORTED;
33afe3
                 break;
33afe3
             }
33afe3
             // fall through
33afe3
@@ -3248,26 +3280,27 @@ determine_op_status(
33afe3
                 pe_proc_err("No further recovery can be attempted for %s "
33afe3
                             "because %s on %s failed (%s%s%s) at %s "
33afe3
                             CRM_XS " rc=%d id=%s", rsc->id, task,
33afe3
-                            node->details->uname, services_ocf_exitcode_str(rc),
33afe3
+                            node->details->uname, services_ocf_exitcode_str(*rc),
33afe3
                             (*exit_reason? ": " : ""), exit_reason,
33afe3
-                            last_change_str(xml_op), rc, ID(xml_op));
33afe3
+                            last_change_str(xml_op), *rc, ID(xml_op));
33afe3
                 pe__clear_resource_flags(rsc, pe_rsc_managed);
33afe3
                 pe__set_resource_flags(rsc, pe_rsc_block);
33afe3
             }
33afe3
-            result = PCMK_EXEC_ERROR_HARD;
33afe3
+            *status = PCMK_EXEC_ERROR_HARD;
33afe3
             break;
33afe3
 
33afe3
         default:
33afe3
-            if (result == PCMK_EXEC_DONE) {
33afe3
+            if (*status == PCMK_EXEC_DONE) {
33afe3
                 crm_info("Treating unknown exit status %d from %s of %s "
33afe3
                          "on %s at %s as failure",
33afe3
-                         rc, task, rsc->id, node->details->uname,
33afe3
+                         *rc, task, rsc->id, node->details->uname,
33afe3
                          last_change_str(xml_op));
33afe3
-                result = PCMK_EXEC_ERROR;
33afe3
+                *status = PCMK_EXEC_ERROR;
33afe3
             }
33afe3
             break;
33afe3
     }
33afe3
-    return result;
33afe3
+
33afe3
+    pe_rsc_trace(rsc, "Remapped %s status to %d", key, *status);
33afe3
 }
33afe3
 
33afe3
 // return TRUE if start or monitor last failure but parameters changed
33afe3
@@ -3622,41 +3655,6 @@ update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, c
33afe3
     }
33afe3
 }
33afe3
 
33afe3
-/*!
33afe3
- * \internal
33afe3
- * \brief Remap informational monitor results to usual values
33afe3
- *
33afe3
- * Certain OCF result codes are for providing extended information to the
33afe3
- * user about services that aren't yet failed but not entirely healthy either.
33afe3
- * These must be treated as the "normal" result by Pacemaker.
33afe3
- *
33afe3
- * \param[in] rc        Actual result of a monitor action
33afe3
- * \param[in] xml_op    Operation history XML
33afe3
- * \param[in] node      Node that operation happened on
33afe3
- * \param[in] rsc       Resource that operation happened to
33afe3
- * \param[in] data_set  Cluster working set
33afe3
- *
33afe3
- * \return Result code that pacemaker should use
33afe3
- *
33afe3
- * \note If the result is remapped, and the node is not shutting down or failed,
33afe3
- *       the operation will be recorded in the data set's list of failed
33afe3
- *       operations, to highlight it for the user.
33afe3
- */
33afe3
-static int
33afe3
-remap_monitor_rc(int rc, xmlNode *xml_op, const pe_node_t *node,
33afe3
-                 const pe_resource_t *rsc, pe_working_set_t *data_set)
33afe3
-{
33afe3
-    int remapped_rc = pcmk__effective_rc(rc);
33afe3
-
33afe3
-    if (rc != remapped_rc) {
33afe3
-        crm_trace("Remapping monitor result %d to %d", rc, remapped_rc);
33afe3
-        if (!node->details->shutdown || node->details->online) {
33afe3
-            record_failed_op(xml_op, node, rsc, data_set);
33afe3
-        }
33afe3
-    }
33afe3
-    return remapped_rc;
33afe3
-}
33afe3
-
33afe3
 static void
33afe3
 unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
33afe3
               xmlNode **last_failure, enum action_fail_response *on_fail,
33afe3
@@ -3712,7 +3710,7 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
33afe3
                      node->details->uname, rsc->id);
33afe3
     }
33afe3
 
33afe3
-    /* It should be possible to call remap_monitor_rc() first then call
33afe3
+    /* It should be possible to call remap_operation() first then call
33afe3
      * check_operation_expiry() only if rc != target_rc, because there should
33afe3
      * never be a fail count without at least one unexpected result in the
33afe3
      * resource history. That would be more efficient by avoiding having to call
33afe3
@@ -3729,9 +3727,8 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
33afe3
         expired = true;
33afe3
     }
33afe3
 
33afe3
-    if (!strcmp(task, CRMD_ACTION_STATUS)) {
33afe3
-        rc = remap_monitor_rc(rc, xml_op, node, rsc, data_set);
33afe3
-    }
33afe3
+    remap_operation(xml_op, rsc, node, data_set, on_fail, target_rc,
33afe3
+                    &rc, &status);
33afe3
 
33afe3
     if (expired && (rc != target_rc)) {
33afe3
         const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
33afe3
@@ -3761,16 +3758,6 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
33afe3
         }
33afe3
     }
33afe3
 
33afe3
-    /* If the executor reported an operation status of anything but done or
33afe3
-     * error, consider that final. But for done or error, we know better whether
33afe3
-     * it should be treated as a failure or not, because we know the expected
33afe3
-     * result.
33afe3
-     */
33afe3
-    if(status == PCMK_EXEC_DONE || status == PCMK_EXEC_ERROR) {
33afe3
-        status = determine_op_status(rsc, rc, target_rc, node, xml_op, on_fail, data_set);
33afe3
-        pe_rsc_trace(rsc, "Remapped %s status to %d", task_key, status);
33afe3
-    }
33afe3
-
33afe3
     switch (status) {
33afe3
         case PCMK_EXEC_CANCELLED:
33afe3
             // Should never happen
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 9fdca1999872b3930cf18b7d807ddb259f23e8a5 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Fri, 19 Nov 2021 15:08:16 -0500
33afe3
Subject: [PATCH 06/21] Test: cts-cli: Add test output for a native resource
33afe3
 with a failed probe op.
33afe3
33afe3
There are no code changes yet to properly handle displaying these
33afe3
operations, so the results here just reflect the current handling.
33afe3
---
33afe3
 cts/cli/crm_mon-partial.xml    | 16 +++++++++++
33afe3
 cts/cli/regression.crm_mon.exp | 50 ++++++++++++++++++++++++++--------
33afe3
 2 files changed, 55 insertions(+), 11 deletions(-)
33afe3
33afe3
diff --git a/cts/cli/crm_mon-partial.xml b/cts/cli/crm_mon-partial.xml
33afe3
index e6c6894b6f..b7817e4775 100644
33afe3
--- a/cts/cli/crm_mon-partial.xml
33afe3
+++ b/cts/cli/crm_mon-partial.xml
33afe3
@@ -60,6 +60,16 @@
33afe3
           </meta_attributes>
33afe3
         </primitive>
33afe3
       </group>
33afe3
+      <primitive class="ocf" id="smart-mon" provider="pacemaker" type="HealthSMART">
33afe3
+        <operations>
33afe3
+          <op id="smart-mon-monitor-interval-10s" interval="10s" name="monitor" start-delay="0s" timeout="10s"/>
33afe3
+          <op id="smart-mon-start-interval-0s" interval="0s" name="start" timeout="10s"/>
33afe3
+          <op id="smart-mon-stop-interval-0s" interval="0s" name="stop" timeout="10s"/>
33afe3
+        </operations>
33afe3
+        <instance_attributes id="smart-mon-instance_attributes">
33afe3
+          <nvpair id="smart-mon-instance_attributes-drives" name="drives" value="/dev/nonexistent"/>
33afe3
+        </instance_attributes>
33afe3
+      </primitive>
33afe3
     </resources>
33afe3
     <constraints/>
33afe3
   </configuration>
33afe3
@@ -94,6 +104,9 @@
33afe3
           <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
33afe3
             <lrm_rsc_op id="dummy-1_last_0" operation_key="dummy-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.6.0" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1599063458" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
           </lrm_resource>
33afe3
+          <lrm_resource id="smart-mon" type="HealthSMART" class="ocf" provider="pacemaker">
33afe3
+            <lrm_rsc_op id="smart-mon_last_failure_0" operation_key="smart-mon_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="9" rc-code="5" op-status="0" interval="0" last-rc-change="1636490335" exec-time="33" queue-time="0" op-digest="b368e619fcd06788c996f6a2ef2efb6a"/>
33afe3
+          </lrm_resource>
33afe3
         </lrm_resources>
33afe3
       </lrm>
33afe3
       <transient_attributes id="2">
33afe3
@@ -135,6 +148,9 @@
33afe3
             <lrm_rsc_op id="httpd-bundle-1_monitor_30000" operation_key="httpd-bundle-1_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="30000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="7592cb10fa1499772a031adfd385f558"/>
33afe3
           </lrm_resource>
33afe3
         </lrm_resources>
33afe3
+        <lrm_resource id="smart-mon" type="HealthSMART" class="ocf" provider="pacemaker">
33afe3
+          <lrm_rsc_op id="smart-mon_last_failure_0" operation_key="smart-mon_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="9" rc-code="5" op-status="0" interval="0" last-rc-change="1636490335" exec-time="33" queue-time="0" op-digest="b368e619fcd06788c996f6a2ef2efb6a"/>
33afe3
+        </lrm_resource>
33afe3
       </lrm>
33afe3
       <transient_attributes id="1">
33afe3
         <instance_attributes id="status-1">
33afe3
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
33afe3
index 8714f917a9..d12dce3ae8 100644
33afe3
--- a/cts/cli/regression.crm_mon.exp
33afe3
+++ b/cts/cli/regression.crm_mon.exp
33afe3
@@ -3470,7 +3470,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 13 resource instances configured (1 DISABLED)
33afe3
+  * 14 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3485,6 +3485,9 @@ Active Resources:
33afe3
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
   * Resource Group: partially-active-group (1 member inactive):
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
+
33afe3
+Failed Resource Actions:
33afe3
+  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources
33afe3
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
33afe3
@@ -3495,7 +3498,7 @@ Active Resources:
33afe3
     <last_update time=""/>
33afe3
     <last_change time=""/>
33afe3
     <nodes_configured number="4"/>
33afe3
-    <resources_configured number="13" disabled="1" blocked="0"/>
33afe3
+    <resources_configured number="14" disabled="1" blocked="0"/>
33afe3
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
33afe3
   </summary>
33afe3
   <nodes>
33afe3
@@ -3548,6 +3551,7 @@ Active Resources:
33afe3
       </resource>
33afe3
       <resource id="dummy-2" resource_agent="ocf:pacemaker:Dummy" role="Stopped" target_role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
33afe3
     </group>
33afe3
+    <resource id="smart-mon" resource_agent="ocf:pacemaker:HealthSMART" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
33afe3
   </resources>
33afe3
   <node_attributes>
33afe3
     <node name="cluster01">
33afe3
@@ -3574,6 +3578,9 @@ Active Resources:
33afe3
       <resource_history id="dummy-1" orphan="false" migration-threshold="1000000">
33afe3
         <operation_history call="2" task="start" rc="0" rc_text="ok" exec-time="0ms" queue-time="0ms"/>
33afe3
       </resource_history>
33afe3
+      <resource_history id="smart-mon" orphan="false" migration-threshold="1000000">
33afe3
+        <operation_history call="9" task="probe" rc="5" rc_text="not installed" exec-time="33ms" queue-time="0ms"/>
33afe3
+      </resource_history>
33afe3
     </node>
33afe3
     <node name="cluster01">
33afe3
       <resource_history id="Fencing" orphan="false" migration-threshold="1000000">
33afe3
@@ -3603,6 +3610,9 @@ Active Resources:
33afe3
       </resource_history>
33afe3
     </node>
33afe3
   </node_history>
33afe3
+  <failures>
33afe3
+    <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
+  </failures>
33afe3
   <status code="0" message="OK"/>
33afe3
 </pacemaker-result>
33afe3
 =#=#=#= End test: XML output of partially active resources - OK (0) =#=#=#=
33afe3
@@ -3614,7 +3624,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 13 resource instances configured (1 DISABLED)
33afe3
+  * 14 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3631,6 +3641,10 @@ Full List of Resources:
33afe3
   * Resource Group: partially-active-group:
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 Stopped (disabled)
33afe3
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
+
33afe3
+Failed Resource Actions:
33afe3
+  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
33afe3
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
33afe3
@@ -3640,13 +3654,14 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 13 resource instances configured (1 DISABLED)
33afe3
+  * 14 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
33afe3
 
33afe3
 Full List of Resources:
33afe3
+  * 0/1	(ocf:pacemaker:HealthSMART):	Active
33afe3
   * 1/1	(stonith:fence_xvm):	Active cluster01
33afe3
   * Clone Set: ping-clone [ping]:
33afe3
     * Started: [ cluster01 ]
33afe3
@@ -3676,6 +3691,8 @@ Operations:
33afe3
       * (3) monitor: interval="30000ms"
33afe3
     * dummy-1: migration-threshold=1000000:
33afe3
       * (2) start
33afe3
+    * smart-mon: migration-threshold=1000000:
33afe3
+      * (9) probe
33afe3
   * Node: cluster01:
33afe3
     * Fencing: migration-threshold=1000000:
33afe3
       * (15) start
33afe3
@@ -3695,6 +3712,9 @@ Operations:
33afe3
   * Node: httpd-bundle-0@cluster02:
33afe3
     * httpd: migration-threshold=1000000:
33afe3
       * (1) start
33afe3
+
33afe3
+Failed Resource Actions:
33afe3
+  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
33afe3
@@ -3704,7 +3724,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 13 resource instances configured (1 DISABLED)
33afe3
+  * 14 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3722,7 +3742,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 13 resource instances configured (1 DISABLED)
33afe3
+  * 14 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3741,7 +3761,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 13 resource instances configured (1 DISABLED)
33afe3
+  * 14 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3759,7 +3779,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 13 resource instances configured (1 DISABLED)
33afe3
+  * 14 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3777,7 +3797,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 13 resource instances configured (1 DISABLED)
33afe3
+  * 14 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Node cluster01: online:
33afe3
@@ -3806,6 +3826,7 @@ Inactive Resources:
33afe3
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
   * Resource Group: partially-active-group:
33afe3
     * 1/2	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
 
33afe3
 Node Attributes:
33afe3
   * Node: cluster01:
33afe3
@@ -3826,6 +3847,8 @@ Operations:
33afe3
       * (3) monitor: interval="30000ms"
33afe3
     * dummy-1: migration-threshold=1000000:
33afe3
       * (2) start
33afe3
+    * smart-mon: migration-threshold=1000000:
33afe3
+      * (9) probe
33afe3
   * Node: cluster01:
33afe3
     * Fencing: migration-threshold=1000000:
33afe3
       * (15) start
33afe3
@@ -3845,6 +3868,9 @@ Operations:
33afe3
   * Node: httpd-bundle-0@cluster02:
33afe3
     * httpd: migration-threshold=1000000:
33afe3
       * (1) start
33afe3
+
33afe3
+Failed Resource Actions:
33afe3
+  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
33afe3
@@ -3854,7 +3880,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 13 resource instances configured (1 DISABLED)
33afe3
+  * 14 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 ]
33afe3
@@ -3865,6 +3891,7 @@ Full List of Resources:
33afe3
   * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
 =#=#=#= End test: Text output of partially active resources, with inactive resources, filtered by node - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources, filtered by node
33afe3
 =#=#=#= Begin test: Text output of partially active resources, filtered by node =#=#=#=
33afe3
@@ -3875,7 +3902,7 @@ Full List of Resources:
33afe3
     <last_update time=""/>
33afe3
     <last_change time=""/>
33afe3
     <nodes_configured number="4"/>
33afe3
-    <resources_configured number="13" disabled="1" blocked="0"/>
33afe3
+    <resources_configured number="14" disabled="1" blocked="0"/>
33afe3
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
33afe3
   </summary>
33afe3
   <nodes>
33afe3
@@ -3905,6 +3932,7 @@ Full List of Resources:
33afe3
         </resource>
33afe3
       </replica>
33afe3
     </bundle>
33afe3
+    <resource id="smart-mon" resource_agent="ocf:pacemaker:HealthSMART" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
33afe3
   </resources>
33afe3
   <node_attributes>
33afe3
     <node name="cluster01">
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 1c54d0bbb74d066d55a56eae28d1a579b8854604 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Fri, 19 Nov 2021 15:17:52 -0500
33afe3
Subject: [PATCH 07/21] Test: cts-cli: Add test output for a cloned resource
33afe3
 with a failed probe op.
33afe3
33afe3
There are no code changes yet to properly handle displaying these
33afe3
operations, so the results here just reflect the current handling.
33afe3
---
33afe3
 cts/cli/crm_mon-partial.xml    |  3 +++
33afe3
 cts/cli/regression.crm_mon.exp | 12 ++++++++++++
33afe3
 2 files changed, 15 insertions(+)
33afe3
33afe3
diff --git a/cts/cli/crm_mon-partial.xml b/cts/cli/crm_mon-partial.xml
33afe3
index b7817e4775..1f9dc156aa 100644
33afe3
--- a/cts/cli/crm_mon-partial.xml
33afe3
+++ b/cts/cli/crm_mon-partial.xml
33afe3
@@ -107,6 +107,9 @@
33afe3
           <lrm_resource id="smart-mon" type="HealthSMART" class="ocf" provider="pacemaker">
33afe3
             <lrm_rsc_op id="smart-mon_last_failure_0" operation_key="smart-mon_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="9" rc-code="5" op-status="0" interval="0" last-rc-change="1636490335" exec-time="33" queue-time="0" op-digest="b368e619fcd06788c996f6a2ef2efb6a"/>
33afe3
           </lrm_resource>
33afe3
+          <lrm_resource id="ping" class="ocf" provider="pacemaker" type="ping">
33afe3
+            <lrm_rsc_op id="ping_last_failure_0" operation_key="ping_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="6" rc-code="5" op-status="0" interval="0" last-rc-change="1637259102" exec-time="0" queue-time="0"/>
33afe3
+          </lrm_resource>
33afe3
         </lrm_resources>
33afe3
       </lrm>
33afe3
       <transient_attributes id="2">
33afe3
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
33afe3
index d12dce3ae8..d093bd8106 100644
33afe3
--- a/cts/cli/regression.crm_mon.exp
33afe3
+++ b/cts/cli/regression.crm_mon.exp
33afe3
@@ -3488,6 +3488,7 @@ Active Resources:
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
+  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources
33afe3
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
33afe3
@@ -3581,6 +3582,9 @@ Failed Resource Actions:
33afe3
       <resource_history id="smart-mon" orphan="false" migration-threshold="1000000">
33afe3
         <operation_history call="9" task="probe" rc="5" rc_text="not installed" exec-time="33ms" queue-time="0ms"/>
33afe3
       </resource_history>
33afe3
+      <resource_history id="ping" orphan="false" migration-threshold="1000000">
33afe3
+        <operation_history call="6" task="probe" rc="5" rc_text="not installed" exec-time="0ms" queue-time="0ms"/>
33afe3
+      </resource_history>
33afe3
     </node>
33afe3
     <node name="cluster01">
33afe3
       <resource_history id="Fencing" orphan="false" migration-threshold="1000000">
33afe3
@@ -3612,6 +3616,7 @@ Failed Resource Actions:
33afe3
   </node_history>
33afe3
   <failures>
33afe3
     <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
+    <failure op_key="ping_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="6" status="complete" last-rc-change="2021-11-18 13:11:42 -05:00" queued="0" exec="0" interval="0" task="monitor"/>
33afe3
   </failures>
33afe3
   <status code="0" message="OK"/>
33afe3
 </pacemaker-result>
33afe3
@@ -3645,6 +3650,7 @@ Full List of Resources:
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
+  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
33afe3
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
33afe3
@@ -3693,6 +3699,8 @@ Operations:
33afe3
       * (2) start
33afe3
     * smart-mon: migration-threshold=1000000:
33afe3
       * (9) probe
33afe3
+    * ping: migration-threshold=1000000:
33afe3
+      * (6) probe
33afe3
   * Node: cluster01:
33afe3
     * Fencing: migration-threshold=1000000:
33afe3
       * (15) start
33afe3
@@ -3715,6 +3723,7 @@ Operations:
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
+  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
33afe3
@@ -3849,6 +3858,8 @@ Operations:
33afe3
       * (2) start
33afe3
     * smart-mon: migration-threshold=1000000:
33afe3
       * (9) probe
33afe3
+    * ping: migration-threshold=1000000:
33afe3
+      * (6) probe
33afe3
   * Node: cluster01:
33afe3
     * Fencing: migration-threshold=1000000:
33afe3
       * (15) start
33afe3
@@ -3871,6 +3882,7 @@ Operations:
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
+  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 9408f08c07eb531ff84b07bf959f3d681ebf2b78 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Fri, 19 Nov 2021 15:48:16 -0500
33afe3
Subject: [PATCH 08/21] Test: cts-cli: Change the resources in
33afe3
 partially-active-group.
33afe3
33afe3
dummy-2 is now not running because it failed to start due to an
33afe3
unimplemented feature.  I don't know what could possibly be
33afe3
unimplemented about a dummy resource, but it's not important.
33afe3
33afe3
There is also a new dummy-3 resource that acts exactly the same as
33afe3
dummy-2.  This preserves checking that the inactive member output can
33afe3
still be displayed.
33afe3
33afe3
There are no code changes yet to properly handle displaying these
33afe3
operations, so the results here just reflect the current handling.
33afe3
---
33afe3
 cts/cli/crm_mon-partial.xml    |  6 +++-
33afe3
 cts/cli/regression.crm_mon.exp | 62 +++++++++++++++++++++++-----------
33afe3
 2 files changed, 47 insertions(+), 21 deletions(-)
33afe3
33afe3
diff --git a/cts/cli/crm_mon-partial.xml b/cts/cli/crm_mon-partial.xml
33afe3
index 1f9dc156aa..1ce80ea58a 100644
33afe3
--- a/cts/cli/crm_mon-partial.xml
33afe3
+++ b/cts/cli/crm_mon-partial.xml
33afe3
@@ -54,7 +54,8 @@
33afe3
       </bundle>
33afe3
       <group id="partially-active-group">
33afe3
         <primitive class="ocf" id="dummy-1" provider="pacemaker" type="Dummy"/>
33afe3
-        <primitive class="ocf" id="dummy-2" provider="pacemaker" type="Dummy">
33afe3
+        <primitive class="ocf" id="dummy-2" provider="pacemaker" type="Dummy"/>
33afe3
+        <primitive class="ocf" id="dummy-3" provider="pacemaker" type="Dummy">
33afe3
           <meta_attributes id="inactive-dummy-meta_attributes">
33afe3
             <nvpair id="inactive-dummy-meta_attributes-target-role" name="target-role" value="Stopped"/>
33afe3
           </meta_attributes>
33afe3
@@ -104,6 +105,9 @@
33afe3
           <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
33afe3
             <lrm_rsc_op id="dummy-1_last_0" operation_key="dummy-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.6.0" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1599063458" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
           </lrm_resource>
33afe3
+          <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:3;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="3" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
           <lrm_resource id="smart-mon" type="HealthSMART" class="ocf" provider="pacemaker">
33afe3
             <lrm_rsc_op id="smart-mon_last_failure_0" operation_key="smart-mon_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="9" rc-code="5" op-status="0" interval="0" last-rc-change="1636490335" exec-time="33" queue-time="0" op-digest="b368e619fcd06788c996f6a2ef2efb6a"/>
33afe3
           </lrm_resource>
33afe3
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
33afe3
index d093bd8106..8cf3a1215e 100644
33afe3
--- a/cts/cli/regression.crm_mon.exp
33afe3
+++ b/cts/cli/regression.crm_mon.exp
33afe3
@@ -3470,7 +3470,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 14 resource instances configured (1 DISABLED)
33afe3
+  * 15 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3485,8 +3485,10 @@ Active Resources:
33afe3
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
   * Resource Group: partially-active-group (1 member inactive):
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
+    * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
+  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
33afe3
@@ -3499,12 +3501,12 @@ Failed Resource Actions:
33afe3
     <last_update time=""/>
33afe3
     <last_change time=""/>
33afe3
     <nodes_configured number="4"/>
33afe3
-    <resources_configured number="14" disabled="1" blocked="0"/>
33afe3
+    <resources_configured number="15" disabled="1" blocked="0"/>
33afe3
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
33afe3
   </summary>
33afe3
   <nodes>
33afe3
     <node name="cluster01" id="1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="false" resources_running="5" type="member"/>
33afe3
-    <node name="cluster02" id="2" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="4" type="member"/>
33afe3
+    <node name="cluster02" id="2" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="5" type="member"/>
33afe3
     <node name="httpd-bundle-0" id="httpd-bundle-0" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="1" type="remote" id_as_resource="httpd-bundle-docker-0"/>
33afe3
     <node name="httpd-bundle-1" id="httpd-bundle-1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="0" type="remote" id_as_resource="httpd-bundle-docker-1"/>
33afe3
   </nodes>
33afe3
@@ -3546,11 +3548,14 @@ Failed Resource Actions:
33afe3
         </resource>
33afe3
       </replica>
33afe3
     </bundle>
33afe3
-    <group id="partially-active-group" number_resources="2" managed="true" disabled="false">
33afe3
+    <group id="partially-active-group" number_resources="3" managed="true" disabled="false">
33afe3
       <resource id="dummy-1" resource_agent="ocf:pacemaker:Dummy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
33afe3
         <node name="cluster02" id="2" cached="true"/>
33afe3
       </resource>
33afe3
-      <resource id="dummy-2" resource_agent="ocf:pacemaker:Dummy" role="Stopped" target_role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
33afe3
+      <resource id="dummy-2" resource_agent="ocf:pacemaker:Dummy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="true" failure_ignored="false" nodes_running_on="1">
33afe3
+        <node name="cluster02" id="2" cached="true"/>
33afe3
+      </resource>
33afe3
+      <resource id="dummy-3" resource_agent="ocf:pacemaker:Dummy" role="Stopped" target_role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
33afe3
     </group>
33afe3
     <resource id="smart-mon" resource_agent="ocf:pacemaker:HealthSMART" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
33afe3
   </resources>
33afe3
@@ -3579,6 +3584,9 @@ Failed Resource Actions:
33afe3
       <resource_history id="dummy-1" orphan="false" migration-threshold="1000000">
33afe3
         <operation_history call="2" task="start" rc="0" rc_text="ok" exec-time="0ms" queue-time="0ms"/>
33afe3
       </resource_history>
33afe3
+      <resource_history id="dummy-2" orphan="false" migration-threshold="1000000">
33afe3
+        <operation_history call="2" task="probe" rc="3" rc_text="unimplemented feature" exec-time="33ms" queue-time="0ms"/>
33afe3
+      </resource_history>
33afe3
       <resource_history id="smart-mon" orphan="false" migration-threshold="1000000">
33afe3
         <operation_history call="9" task="probe" rc="5" rc_text="not installed" exec-time="33ms" queue-time="0ms"/>
33afe3
       </resource_history>
33afe3
@@ -3615,6 +3623,7 @@ Failed Resource Actions:
33afe3
     </node>
33afe3
   </node_history>
33afe3
   <failures>
33afe3
+    <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
     <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
     <failure op_key="ping_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="6" status="complete" last-rc-change="2021-11-18 13:11:42 -05:00" queued="0" exec="0" interval="0" task="monitor"/>
33afe3
   </failures>
33afe3
@@ -3629,7 +3638,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 14 resource instances configured (1 DISABLED)
33afe3
+  * 15 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3645,10 +3654,12 @@ Full List of Resources:
33afe3
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
   * Resource Group: partially-active-group:
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
-    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped (disabled)
33afe3
+    * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
+    * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
33afe3
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
+  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
33afe3
@@ -3660,7 +3671,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 14 resource instances configured (1 DISABLED)
33afe3
+  * 15 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3676,7 +3687,7 @@ Full List of Resources:
33afe3
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
   * Resource Group: partially-active-group:
33afe3
-    * 1/2	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
+    * 2/3	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
 
33afe3
 Node Attributes:
33afe3
   * Node: cluster01:
33afe3
@@ -3697,6 +3708,8 @@ Operations:
33afe3
       * (3) monitor: interval="30000ms"
33afe3
     * dummy-1: migration-threshold=1000000:
33afe3
       * (2) start
33afe3
+    * dummy-2: migration-threshold=1000000:
33afe3
+      * (2) probe
33afe3
     * smart-mon: migration-threshold=1000000:
33afe3
       * (9) probe
33afe3
     * ping: migration-threshold=1000000:
33afe3
@@ -3722,6 +3735,7 @@ Operations:
33afe3
       * (1) start
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
+  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
33afe3
@@ -3733,7 +3747,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 14 resource instances configured (1 DISABLED)
33afe3
+  * 15 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3742,6 +3756,7 @@ Node List:
33afe3
 Active Resources:
33afe3
   * Resource Group: partially-active-group (1 member inactive):
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
+    * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
 =#=#=#= End test: Text output of partially active group - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active group
33afe3
 =#=#=#= Begin test: Text output of partially active group, with inactive resources =#=#=#=
33afe3
@@ -3751,7 +3766,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 14 resource instances configured (1 DISABLED)
33afe3
+  * 15 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3760,7 +3775,8 @@ Node List:
33afe3
 Full List of Resources:
33afe3
   * Resource Group: partially-active-group:
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
-    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped (disabled)
33afe3
+    * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
+    * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
33afe3
 =#=#=#= End test: Text output of partially active group, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active group, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of active member of partially active group =#=#=#=
33afe3
@@ -3770,7 +3786,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 14 resource instances configured (1 DISABLED)
33afe3
+  * 15 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3788,7 +3804,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 14 resource instances configured (1 DISABLED)
33afe3
+  * 15 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3796,7 +3812,10 @@ Node List:
33afe3
 
33afe3
 Active Resources:
33afe3
   * Resource Group: partially-active-group (1 member inactive):
33afe3
-    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped (disabled)
33afe3
+    * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
+
33afe3
+Failed Resource Actions:
33afe3
+  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
 =#=#=#= End test: Text output of inactive member of partially active group - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of inactive member of partially active group
33afe3
 =#=#=#= Begin test: Complete brief text output grouped by node, with inactive resources =#=#=#=
33afe3
@@ -3806,7 +3825,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 14 resource instances configured (1 DISABLED)
33afe3
+  * 15 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Node cluster01: online:
33afe3
@@ -3820,7 +3839,7 @@ Node List:
33afe3
     * Resources:
33afe3
       * 1	(ocf:heartbeat:IPaddr2):	Active 
33afe3
       * 1	(ocf:heartbeat:docker):	Active 
33afe3
-      * 1	(ocf:pacemaker:Dummy):	Active 
33afe3
+      * 2	(ocf:pacemaker:Dummy):	Active 
33afe3
       * 1	(ocf:pacemaker:remote):	Active 
33afe3
   * GuestNode httpd-bundle-0@cluster02: online:
33afe3
     * Resources:
33afe3
@@ -3834,7 +3853,7 @@ Inactive Resources:
33afe3
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
   * Resource Group: partially-active-group:
33afe3
-    * 1/2	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
+    * 2/3	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
 
33afe3
 Node Attributes:
33afe3
@@ -3856,6 +3875,8 @@ Operations:
33afe3
       * (3) monitor: interval="30000ms"
33afe3
     * dummy-1: migration-threshold=1000000:
33afe3
       * (2) start
33afe3
+    * dummy-2: migration-threshold=1000000:
33afe3
+      * (2) probe
33afe3
     * smart-mon: migration-threshold=1000000:
33afe3
       * (9) probe
33afe3
     * ping: migration-threshold=1000000:
33afe3
@@ -3881,6 +3902,7 @@ Operations:
33afe3
       * (1) start
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
+  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
33afe3
@@ -3892,7 +3914,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 14 resource instances configured (1 DISABLED)
33afe3
+  * 15 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 ]
33afe3
@@ -3914,7 +3936,7 @@ Full List of Resources:
33afe3
     <last_update time=""/>
33afe3
     <last_change time=""/>
33afe3
     <nodes_configured number="4"/>
33afe3
-    <resources_configured number="14" disabled="1" blocked="0"/>
33afe3
+    <resources_configured number="15" disabled="1" blocked="0"/>
33afe3
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
33afe3
   </summary>
33afe3
   <nodes>
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 85e76b8bdb4de261a9cb4858eeedd49fba0346a1 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Fri, 19 Nov 2021 15:55:51 -0500
33afe3
Subject: [PATCH 09/21] Test: cts-cli: Add a failed probe on a new dummy-4
33afe3
 resource.
33afe3
33afe3
This is to verify that these resources which are part of a group are
33afe3
displayed properly.  No code changes will be necessary, since groups are
33afe3
just several other resources all in the same pile.
33afe3
33afe3
There are no code changes yet to properly handle displaying these
33afe3
operations, so the results here just reflect the current handling.
33afe3
---
33afe3
 cts/cli/crm_mon-partial.xml    |  4 +++
33afe3
 cts/cli/regression.crm_mon.exp | 51 ++++++++++++++++++++++------------
33afe3
 2 files changed, 37 insertions(+), 18 deletions(-)
33afe3
33afe3
diff --git a/cts/cli/crm_mon-partial.xml b/cts/cli/crm_mon-partial.xml
33afe3
index 1ce80ea58a..d4d4a70848 100644
33afe3
--- a/cts/cli/crm_mon-partial.xml
33afe3
+++ b/cts/cli/crm_mon-partial.xml
33afe3
@@ -60,6 +60,7 @@
33afe3
             <nvpair id="inactive-dummy-meta_attributes-target-role" name="target-role" value="Stopped"/>
33afe3
           </meta_attributes>
33afe3
         </primitive>
33afe3
+        <primitive class="ocf" id="dummy-4" provider="pacemaker" type="Dummy"/>
33afe3
       </group>
33afe3
       <primitive class="ocf" id="smart-mon" provider="pacemaker" type="HealthSMART">
33afe3
         <operations>
33afe3
@@ -108,6 +109,9 @@
33afe3
           <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
33afe3
             <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:3;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="3" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
           </lrm_resource>
33afe3
+          <lrm_resource id="dummy-4" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-4_last_failure_0" operation_key="dummy-4_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="21:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;21:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
           <lrm_resource id="smart-mon" type="HealthSMART" class="ocf" provider="pacemaker">
33afe3
             <lrm_rsc_op id="smart-mon_last_failure_0" operation_key="smart-mon_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="9" rc-code="5" op-status="0" interval="0" last-rc-change="1636490335" exec-time="33" queue-time="0" op-digest="b368e619fcd06788c996f6a2ef2efb6a"/>
33afe3
           </lrm_resource>
33afe3
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
33afe3
index 8cf3a1215e..c524b199e3 100644
33afe3
--- a/cts/cli/regression.crm_mon.exp
33afe3
+++ b/cts/cli/regression.crm_mon.exp
33afe3
@@ -3470,7 +3470,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 15 resource instances configured (1 DISABLED)
33afe3
+  * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3483,12 +3483,13 @@ Active Resources:
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
-  * Resource Group: partially-active-group (1 member inactive):
33afe3
+  * Resource Group: partially-active-group (2 members inactive):
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
+  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
33afe3
@@ -3501,7 +3502,7 @@ Failed Resource Actions:
33afe3
     <last_update time=""/>
33afe3
     <last_change time=""/>
33afe3
     <nodes_configured number="4"/>
33afe3
-    <resources_configured number="15" disabled="1" blocked="0"/>
33afe3
+    <resources_configured number="16" disabled="1" blocked="0"/>
33afe3
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
33afe3
   </summary>
33afe3
   <nodes>
33afe3
@@ -3548,7 +3549,7 @@ Failed Resource Actions:
33afe3
         </resource>
33afe3
       </replica>
33afe3
     </bundle>
33afe3
-    <group id="partially-active-group" number_resources="3" managed="true" disabled="false">
33afe3
+    <group id="partially-active-group" number_resources="4" managed="true" disabled="false">
33afe3
       <resource id="dummy-1" resource_agent="ocf:pacemaker:Dummy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
33afe3
         <node name="cluster02" id="2" cached="true"/>
33afe3
       </resource>
33afe3
@@ -3556,6 +3557,7 @@ Failed Resource Actions:
33afe3
         <node name="cluster02" id="2" cached="true"/>
33afe3
       </resource>
33afe3
       <resource id="dummy-3" resource_agent="ocf:pacemaker:Dummy" role="Stopped" target_role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
33afe3
+      <resource id="dummy-4" resource_agent="ocf:pacemaker:Dummy" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
33afe3
     </group>
33afe3
     <resource id="smart-mon" resource_agent="ocf:pacemaker:HealthSMART" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
33afe3
   </resources>
33afe3
@@ -3587,6 +3589,9 @@ Failed Resource Actions:
33afe3
       <resource_history id="dummy-2" orphan="false" migration-threshold="1000000">
33afe3
         <operation_history call="2" task="probe" rc="3" rc_text="unimplemented feature" exec-time="33ms" queue-time="0ms"/>
33afe3
       </resource_history>
33afe3
+      <resource_history id="dummy-4" orphan="false" migration-threshold="1000000">
33afe3
+        <operation_history call="2" task="probe" rc="5" rc_text="not installed" exec-time="0ms" queue-time="0ms"/>
33afe3
+      </resource_history>
33afe3
       <resource_history id="smart-mon" orphan="false" migration-threshold="1000000">
33afe3
         <operation_history call="9" task="probe" rc="5" rc_text="not installed" exec-time="33ms" queue-time="0ms"/>
33afe3
       </resource_history>
33afe3
@@ -3624,6 +3629,7 @@ Failed Resource Actions:
33afe3
   </node_history>
33afe3
   <failures>
33afe3
     <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
+    <failure op_key="dummy-4_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="0" interval="0" task="monitor"/>
33afe3
     <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
     <failure op_key="ping_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="6" status="complete" last-rc-change="2021-11-18 13:11:42 -05:00" queued="0" exec="0" interval="0" task="monitor"/>
33afe3
   </failures>
33afe3
@@ -3638,7 +3644,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 15 resource instances configured (1 DISABLED)
33afe3
+  * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3656,10 +3662,12 @@ Full List of Resources:
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
     * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
33afe3
+    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped
33afe3
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
+  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
33afe3
@@ -3671,7 +3679,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 15 resource instances configured (1 DISABLED)
33afe3
+  * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3687,7 +3695,7 @@ Full List of Resources:
33afe3
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
   * Resource Group: partially-active-group:
33afe3
-    * 2/3	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
+    * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
 
33afe3
 Node Attributes:
33afe3
   * Node: cluster01:
33afe3
@@ -3710,6 +3718,8 @@ Operations:
33afe3
       * (2) start
33afe3
     * dummy-2: migration-threshold=1000000:
33afe3
       * (2) probe
33afe3
+    * dummy-4: migration-threshold=1000000:
33afe3
+      * (2) probe
33afe3
     * smart-mon: migration-threshold=1000000:
33afe3
       * (9) probe
33afe3
     * ping: migration-threshold=1000000:
33afe3
@@ -3736,6 +3746,7 @@ Operations:
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
+  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
33afe3
@@ -3747,14 +3758,14 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 15 resource instances configured (1 DISABLED)
33afe3
+  * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
33afe3
 
33afe3
 Active Resources:
33afe3
-  * Resource Group: partially-active-group (1 member inactive):
33afe3
+  * Resource Group: partially-active-group (2 members inactive):
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
 =#=#=#= End test: Text output of partially active group - OK (0) =#=#=#=
33afe3
@@ -3766,7 +3777,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 15 resource instances configured (1 DISABLED)
33afe3
+  * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
@@ -3777,6 +3788,7 @@ Full List of Resources:
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
     * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
33afe3
+    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped
33afe3
 =#=#=#= End test: Text output of partially active group, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active group, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of active member of partially active group =#=#=#=
33afe3
@@ -3786,14 +3798,14 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 15 resource instances configured (1 DISABLED)
33afe3
+  * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
33afe3
 
33afe3
 Active Resources:
33afe3
-  * Resource Group: partially-active-group (1 member inactive):
33afe3
+  * Resource Group: partially-active-group (2 members inactive):
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
 =#=#=#= End test: Text output of active member of partially active group - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of active member of partially active group
33afe3
@@ -3804,14 +3816,14 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 15 resource instances configured (1 DISABLED)
33afe3
+  * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 cluster02 ]
33afe3
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
33afe3
 
33afe3
 Active Resources:
33afe3
-  * Resource Group: partially-active-group (1 member inactive):
33afe3
+  * Resource Group: partially-active-group (2 members inactive):
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
@@ -3825,7 +3837,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 15 resource instances configured (1 DISABLED)
33afe3
+  * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Node cluster01: online:
33afe3
@@ -3853,7 +3865,7 @@ Inactive Resources:
33afe3
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
   * Resource Group: partially-active-group:
33afe3
-    * 2/3	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
+    * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
 
33afe3
 Node Attributes:
33afe3
@@ -3877,6 +3889,8 @@ Operations:
33afe3
       * (2) start
33afe3
     * dummy-2: migration-threshold=1000000:
33afe3
       * (2) probe
33afe3
+    * dummy-4: migration-threshold=1000000:
33afe3
+      * (2) probe
33afe3
     * smart-mon: migration-threshold=1000000:
33afe3
       * (9) probe
33afe3
     * ping: migration-threshold=1000000:
33afe3
@@ -3903,6 +3917,7 @@ Operations:
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
+  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
33afe3
@@ -3914,7 +3929,7 @@ Cluster Summary:
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
-  * 15 resource instances configured (1 DISABLED)
33afe3
+  * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
   * Online: [ cluster01 ]
33afe3
@@ -3936,7 +3951,7 @@ Full List of Resources:
33afe3
     <last_update time=""/>
33afe3
     <last_change time=""/>
33afe3
     <nodes_configured number="4"/>
33afe3
-    <resources_configured number="15" disabled="1" blocked="0"/>
33afe3
+    <resources_configured number="16" disabled="1" blocked="0"/>
33afe3
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
33afe3
   </summary>
33afe3
   <nodes>
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 206d733b6ce8e0ffcad243d282e8baa8c3ff72b4 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Tue, 23 Nov 2021 14:33:47 -0500
33afe3
Subject: [PATCH 10/21] Test: cts-cli: Add test output for a bundle resource
33afe3
 with a failed probe op.
33afe3
33afe3
This just changes the existing failed bundle resource from not starting
33afe3
to failing with a reason.
33afe3
33afe3
There are no code changes yet to properly handle displaying these
33afe3
operations, so the results here just reflect the current handling.
33afe3
---
33afe3
 cts/cli/crm_mon-partial.xml    |  9 ++++++++
33afe3
 cts/cli/regression.crm_mon.exp | 40 +++++++++++++++++++++++++---------
33afe3
 2 files changed, 39 insertions(+), 10 deletions(-)
33afe3
33afe3
diff --git a/cts/cli/crm_mon-partial.xml b/cts/cli/crm_mon-partial.xml
33afe3
index d4d4a70848..5981fc653c 100644
33afe3
--- a/cts/cli/crm_mon-partial.xml
33afe3
+++ b/cts/cli/crm_mon-partial.xml
33afe3
@@ -178,5 +178,14 @@
33afe3
         </lrm_resources>
33afe3
       </lrm>
33afe3
     </node_state>
33afe3
+    <node_state id="httpd-bundle-1" uname="httpd-bundle-1">
33afe3
+      <lrm id="httpd-bundle-1">
33afe3
+        <lrm_resources>
33afe3
+          <lrm_resource id="httpd" class="ocf" provider="heartbeat" type="apache">
33afe3
+            <lrm_rsc_op id="httpd_last_failure_0" operation_key="httpd_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="1:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:2;1:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="2" op-status="0" interval="0" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+        </lrm_resources>
33afe3
+      </lrm>
33afe3
+    </node_state>
33afe3
   </status>
33afe3
 </cib>
33afe3
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
33afe3
index c524b199e3..b690a26fb6 100644
33afe3
--- a/cts/cli/regression.crm_mon.exp
33afe3
+++ b/cts/cli/regression.crm_mon.exp
33afe3
@@ -3482,7 +3482,7 @@ Active Resources:
33afe3
   * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
+    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
33afe3
   * Resource Group: partially-active-group (2 members inactive):
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
@@ -3492,6 +3492,7 @@ Failed Resource Actions:
33afe3
   * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
+  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
33afe3
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources
33afe3
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
33afe3
@@ -3509,7 +3510,7 @@ Failed Resource Actions:
33afe3
     <node name="cluster01" id="1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="false" resources_running="5" type="member"/>
33afe3
     <node name="cluster02" id="2" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="5" type="member"/>
33afe3
     <node name="httpd-bundle-0" id="httpd-bundle-0" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="1" type="remote" id_as_resource="httpd-bundle-docker-0"/>
33afe3
-    <node name="httpd-bundle-1" id="httpd-bundle-1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="0" type="remote" id_as_resource="httpd-bundle-docker-1"/>
33afe3
+    <node name="httpd-bundle-1" id="httpd-bundle-1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="1" type="remote" id_as_resource="httpd-bundle-docker-1"/>
33afe3
   </nodes>
33afe3
   <resources>
33afe3
     <clone id="ping-clone" multi_state="false" unique="false" managed="true" disabled="false" failed="false" failure_ignored="false">
33afe3
@@ -3540,7 +3541,9 @@ Failed Resource Actions:
33afe3
         <resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf:heartbeat:IPaddr2" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
33afe3
           <node name="cluster01" id="1" cached="true"/>
33afe3
         </resource>
33afe3
-        <resource id="httpd" resource_agent="ocf:heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
33afe3
+        <resource id="httpd" resource_agent="ocf:heartbeat:apache" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="true" failure_ignored="false" nodes_running_on="1">
33afe3
+          <node name="httpd-bundle-1" id="httpd-bundle-1" cached="true"/>
33afe3
+        </resource>
33afe3
         <resource id="httpd-bundle-docker-1" resource_agent="ocf:heartbeat:docker" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
33afe3
           <node name="cluster01" id="1" cached="true"/>
33afe3
         </resource>
33afe3
@@ -3626,12 +3629,18 @@ Failed Resource Actions:
33afe3
         <operation_history call="1" task="start" rc="0" rc_text="ok" exec-time="0ms" queue-time="0ms"/>
33afe3
       </resource_history>
33afe3
     </node>
33afe3
+    <node name="httpd-bundle-1">
33afe3
+      <resource_history id="httpd" orphan="false" migration-threshold="1000000">
33afe3
+        <operation_history call="1" task="probe" rc="2" rc_text="invalid parameter" exec-time="0ms" queue-time="0ms"/>
33afe3
+      </resource_history>
33afe3
+    </node>
33afe3
   </node_history>
33afe3
   <failures>
33afe3
     <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
     <failure op_key="dummy-4_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="0" interval="0" task="monitor"/>
33afe3
     <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
     <failure op_key="ping_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="6" status="complete" last-rc-change="2021-11-18 13:11:42 -05:00" queued="0" exec="0" interval="0" task="monitor"/>
33afe3
+    <failure op_key="httpd_monitor_0" node="httpd-bundle-1" exitstatus="invalid parameter" exitreason="" exitcode="2" call="1" status="complete" last-rc-change="2020-05-27 15:43:09 -04:00" queued="0" exec="0" interval="0" task="monitor"/>
33afe3
   </failures>
33afe3
   <status code="0" message="OK"/>
33afe3
 </pacemaker-result>
33afe3
@@ -3657,7 +3666,7 @@ Full List of Resources:
33afe3
   * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
+    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
33afe3
   * Resource Group: partially-active-group:
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
@@ -3670,6 +3679,7 @@ Failed Resource Actions:
33afe3
   * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
+  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
33afe3
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
33afe3
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
33afe3
@@ -3693,7 +3703,7 @@ Full List of Resources:
33afe3
     * Stopped: [ cluster02 ]
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
+    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
33afe3
   * Resource Group: partially-active-group:
33afe3
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
 
33afe3
@@ -3743,12 +3753,16 @@ Operations:
33afe3
   * Node: httpd-bundle-0@cluster02:
33afe3
     * httpd: migration-threshold=1000000:
33afe3
       * (1) start
33afe3
+  * Node: httpd-bundle-1@cluster01:
33afe3
+    * httpd: migration-threshold=1000000:
33afe3
+      * (1) probe
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
   * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
+  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
33afe3
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
33afe3
@@ -3856,14 +3870,14 @@ Node List:
33afe3
   * GuestNode httpd-bundle-0@cluster02: online:
33afe3
     * Resources:
33afe3
       * 1	(ocf:heartbeat:apache):	Active 
33afe3
+  * GuestNode httpd-bundle-1@cluster01: online:
33afe3
+    * Resources:
33afe3
+      * 1	(ocf:heartbeat:apache):	Active 
33afe3
 
33afe3
 Inactive Resources:
33afe3
   * Clone Set: ping-clone [ping]:
33afe3
     * Started: [ cluster01 ]
33afe3
     * Stopped: [ cluster02 ]
33afe3
-  * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
-    * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
   * Resource Group: partially-active-group:
33afe3
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
@@ -3914,12 +3928,16 @@ Operations:
33afe3
   * Node: httpd-bundle-0@cluster02:
33afe3
     * httpd: migration-threshold=1000000:
33afe3
       * (1) start
33afe3
+  * Node: httpd-bundle-1@cluster01:
33afe3
+    * httpd: migration-threshold=1000000:
33afe3
+      * (1) probe
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
   * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
+  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
33afe3
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
33afe3
@@ -3939,7 +3957,7 @@ Full List of Resources:
33afe3
     * Started: [ cluster01 ]
33afe3
   * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
33afe3
+    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
33afe3
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
 =#=#=#= End test: Text output of partially active resources, with inactive resources, filtered by node - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources, filtered by node
33afe3
@@ -3972,7 +3990,9 @@ Full List of Resources:
33afe3
         <resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf:heartbeat:IPaddr2" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
33afe3
           <node name="cluster01" id="1" cached="true"/>
33afe3
         </resource>
33afe3
-        <resource id="httpd" resource_agent="ocf:heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
33afe3
+        <resource id="httpd" resource_agent="ocf:heartbeat:apache" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="true" failure_ignored="false" nodes_running_on="1">
33afe3
+          <node name="httpd-bundle-1" id="httpd-bundle-1" cached="true"/>
33afe3
+        </resource>
33afe3
         <resource id="httpd-bundle-docker-1" resource_agent="ocf:heartbeat:docker" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
33afe3
           <node name="cluster01" id="1" cached="true"/>
33afe3
         </resource>
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 6240a28d36c0349e3b1d7f52c36106580c53bb01 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Mon, 22 Nov 2021 10:59:10 -0500
33afe3
Subject: [PATCH 11/21] Test: cts: Add --show-detail to a couple of the crm_mon
33afe3
 tests.
33afe3
33afe3
This straightens out a couple differences in output between running
33afe3
tests locally (where --enable-compat-2.0 is not given, which would
33afe3
automatically add --show-detail) and running tests under mock (where
33afe3
that option is given).
33afe3
33afe3
Note that this only really matters for failed resource actions, which
33afe3
were not previously output as part of any crm_mon regression test.  It
33afe3
is only the patches in this series that have introduced those, and thus
33afe3
this difference.
33afe3
---
33afe3
 cts/cli/regression.crm_mon.exp | 131 ++++++++++++++++++++-------------
33afe3
 cts/cts-cli.in                 |  10 +--
33afe3
 2 files changed, 83 insertions(+), 58 deletions(-)
33afe3
33afe3
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
33afe3
index b690a26fb6..d7b9d98e2c 100644
33afe3
--- a/cts/cli/regression.crm_mon.exp
33afe3
+++ b/cts/cli/regression.crm_mon.exp
33afe3
@@ -3466,33 +3466,42 @@ Operations:
33afe3
 =#=#=#= Begin test: Text output of partially active resources =#=#=#=
33afe3
 Cluster Summary:
33afe3
   * Stack: corosync
33afe3
-  * Current DC: cluster02 (version) - partition with quorum
33afe3
+  * Current DC: cluster02 (2) (version) - partition with quorum
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
   * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
-  * Online: [ cluster01 cluster02 ]
33afe3
+  * Online: [ cluster01 (1) cluster02 (2) ]
33afe3
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
33afe3
 
33afe3
 Active Resources:
33afe3
   * Clone Set: ping-clone [ping]:
33afe3
-    * Started: [ cluster01 ]
33afe3
+    * ping	(ocf:pacemaker:ping):	 Started cluster01
33afe3
+    * ping	(ocf:pacemaker:ping):	 Stopped
33afe3
   * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
-    * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
33afe3
+    * Replica[0]
33afe3
+      * httpd-bundle-ip-192.168.122.131	(ocf:heartbeat:IPaddr2):	 Started cluster02
33afe3
+      * httpd	(ocf:heartbeat:apache):	 Started httpd-bundle-0
33afe3
+      * httpd-bundle-docker-0	(ocf:heartbeat:docker):	 Started cluster02
33afe3
+      * httpd-bundle-0	(ocf:pacemaker:remote):	 Started cluster02
33afe3
+    * Replica[1]
33afe3
+      * httpd-bundle-ip-192.168.122.132	(ocf:heartbeat:IPaddr2):	 Started cluster01
33afe3
+      * httpd	(ocf:heartbeat:apache):	 FAILED httpd-bundle-1
33afe3
+      * httpd-bundle-docker-1	(ocf:heartbeat:docker):	 Started cluster01
33afe3
+      * httpd-bundle-1	(ocf:pacemaker:remote):	 Started cluster01
33afe3
   * Resource Group: partially-active-group (2 members inactive):
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
-  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
-  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
-  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
-  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
-  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
33afe3
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
+  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
33afe3
+  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
33afe3
+  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
33afe3
+  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
33afe3
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources
33afe3
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
33afe3
@@ -3649,24 +3658,32 @@ Failed Resource Actions:
33afe3
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources =#=#=#=
33afe3
 Cluster Summary:
33afe3
   * Stack: corosync
33afe3
-  * Current DC: cluster02 (version) - partition with quorum
33afe3
+  * Current DC: cluster02 (2) (version) - partition with quorum
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
   * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
-  * Online: [ cluster01 cluster02 ]
33afe3
+  * Online: [ cluster01 (1) cluster02 (2) ]
33afe3
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
33afe3
 
33afe3
 Full List of Resources:
33afe3
   * Clone Set: ping-clone [ping]:
33afe3
-    * Started: [ cluster01 ]
33afe3
-    * Stopped: [ cluster02 ]
33afe3
+    * ping	(ocf:pacemaker:ping):	 Started cluster01
33afe3
+    * ping	(ocf:pacemaker:ping):	 Stopped
33afe3
   * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
-    * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
33afe3
+    * Replica[0]
33afe3
+      * httpd-bundle-ip-192.168.122.131	(ocf:heartbeat:IPaddr2):	 Started cluster02
33afe3
+      * httpd	(ocf:heartbeat:apache):	 Started httpd-bundle-0
33afe3
+      * httpd-bundle-docker-0	(ocf:heartbeat:docker):	 Started cluster02
33afe3
+      * httpd-bundle-0	(ocf:pacemaker:remote):	 Started cluster02
33afe3
+    * Replica[1]
33afe3
+      * httpd-bundle-ip-192.168.122.132	(ocf:heartbeat:IPaddr2):	 Started cluster01
33afe3
+      * httpd	(ocf:heartbeat:apache):	 FAILED httpd-bundle-1
33afe3
+      * httpd-bundle-docker-1	(ocf:heartbeat:docker):	 Started cluster01
33afe3
+      * httpd-bundle-1	(ocf:pacemaker:remote):	 Started cluster01
33afe3
   * Resource Group: partially-active-group:
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
@@ -3675,46 +3692,54 @@ Full List of Resources:
33afe3
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
-  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
-  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
-  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
-  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
-  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
33afe3
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
+  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
33afe3
+  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
33afe3
+  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
33afe3
+  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
33afe3
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
33afe3
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
33afe3
 Cluster Summary:
33afe3
   * Stack: corosync
33afe3
-  * Current DC: cluster02 (version) - partition with quorum
33afe3
+  * Current DC: cluster02 (2) (version) - partition with quorum
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
   * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
-  * Online: [ cluster01 cluster02 ]
33afe3
+  * Online: [ cluster01 (1) cluster02 (2) ]
33afe3
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
33afe3
 
33afe3
 Full List of Resources:
33afe3
   * 0/1	(ocf:pacemaker:HealthSMART):	Active
33afe3
   * 1/1	(stonith:fence_xvm):	Active cluster01
33afe3
   * Clone Set: ping-clone [ping]:
33afe3
-    * Started: [ cluster01 ]
33afe3
-    * Stopped: [ cluster02 ]
33afe3
+    * ping	(ocf:pacemaker:ping):	 Started cluster01
33afe3
+    * ping	(ocf:pacemaker:ping):	 Stopped
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
-    * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
33afe3
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
33afe3
+    * Replica[0]
33afe3
+      * httpd-bundle-ip-192.168.122.131	(ocf:heartbeat:IPaddr2):	 Started cluster02
33afe3
+      * httpd	(ocf:heartbeat:apache):	 Started httpd-bundle-0
33afe3
+      * httpd-bundle-docker-0	(ocf:heartbeat:docker):	 Started cluster02
33afe3
+      * httpd-bundle-0	(ocf:pacemaker:remote):	 Started cluster02
33afe3
+    * Replica[1]
33afe3
+      * httpd-bundle-ip-192.168.122.132	(ocf:heartbeat:IPaddr2):	 Started cluster01
33afe3
+      * httpd	(ocf:heartbeat:apache):	 FAILED httpd-bundle-1
33afe3
+      * httpd-bundle-docker-1	(ocf:heartbeat:docker):	 Started cluster01
33afe3
+      * httpd-bundle-1	(ocf:pacemaker:remote):	 Started cluster01
33afe3
   * Resource Group: partially-active-group:
33afe3
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
 
33afe3
 Node Attributes:
33afe3
-  * Node: cluster01:
33afe3
+  * Node: cluster01 (1):
33afe3
     * pingd                           	: 1000      
33afe3
-  * Node: cluster02:
33afe3
+  * Node: cluster02 (2):
33afe3
     * pingd                           	: 1000      
33afe3
 
33afe3
 Operations:
33afe3
-  * Node: cluster02:
33afe3
+  * Node: cluster02 (2):
33afe3
     * httpd-bundle-ip-192.168.122.131: migration-threshold=1000000:
33afe3
       * (2) start
33afe3
       * (3) monitor: interval="60000ms"
33afe3
@@ -3734,7 +3759,7 @@ Operations:
33afe3
       * (9) probe
33afe3
     * ping: migration-threshold=1000000:
33afe3
       * (6) probe
33afe3
-  * Node: cluster01:
33afe3
+  * Node: cluster01 (1):
33afe3
     * Fencing: migration-threshold=1000000:
33afe3
       * (15) start
33afe3
       * (20) monitor: interval="60000ms"
33afe3
@@ -3758,11 +3783,11 @@ Operations:
33afe3
       * (1) probe
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
-  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
-  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
-  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
-  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
-  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
33afe3
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
+  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
33afe3
+  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
33afe3
+  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
33afe3
+  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
33afe3
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
33afe3
@@ -3826,14 +3851,14 @@ Active Resources:
33afe3
 =#=#=#= Begin test: Text output of inactive member of partially active group =#=#=#=
33afe3
 Cluster Summary:
33afe3
   * Stack: corosync
33afe3
-  * Current DC: cluster02 (version) - partition with quorum
33afe3
+  * Current DC: cluster02 (2) (version) - partition with quorum
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
   * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
-  * Online: [ cluster01 cluster02 ]
33afe3
+  * Online: [ cluster01 (1) cluster02 (2) ]
33afe3
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
33afe3
 
33afe3
 Active Resources:
33afe3
@@ -3841,27 +3866,27 @@ Active Resources:
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
-  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
 =#=#=#= End test: Text output of inactive member of partially active group - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of inactive member of partially active group
33afe3
 =#=#=#= Begin test: Complete brief text output grouped by node, with inactive resources =#=#=#=
33afe3
 Cluster Summary:
33afe3
   * Stack: corosync
33afe3
-  * Current DC: cluster02 (version) - partition with quorum
33afe3
+  * Current DC: cluster02 (2) (version) - partition with quorum
33afe3
   * Last updated:
33afe3
   * Last change:
33afe3
   * 4 nodes configured
33afe3
   * 16 resource instances configured (1 DISABLED)
33afe3
 
33afe3
 Node List:
33afe3
-  * Node cluster01: online:
33afe3
+  * Node cluster01 (1): online:
33afe3
     * Resources:
33afe3
       * 1	(ocf:heartbeat:IPaddr2):	Active 
33afe3
       * 1	(ocf:heartbeat:docker):	Active 
33afe3
       * 1	(ocf:pacemaker:ping):	Active 
33afe3
       * 1	(ocf:pacemaker:remote):	Active 
33afe3
       * 1	(stonith:fence_xvm):	Active 
33afe3
-  * Node cluster02: online:
33afe3
+  * Node cluster02 (2): online:
33afe3
     * Resources:
33afe3
       * 1	(ocf:heartbeat:IPaddr2):	Active 
33afe3
       * 1	(ocf:heartbeat:docker):	Active 
33afe3
@@ -3876,20 +3901,20 @@ Node List:
33afe3
 
33afe3
 Inactive Resources:
33afe3
   * Clone Set: ping-clone [ping]:
33afe3
-    * Started: [ cluster01 ]
33afe3
-    * Stopped: [ cluster02 ]
33afe3
+    * ping	(ocf:pacemaker:ping):	 Started cluster01
33afe3
+    * ping	(ocf:pacemaker:ping):	 Stopped
33afe3
   * Resource Group: partially-active-group:
33afe3
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
 
33afe3
 Node Attributes:
33afe3
-  * Node: cluster01:
33afe3
+  * Node: cluster01 (1):
33afe3
     * pingd                           	: 1000      
33afe3
-  * Node: cluster02:
33afe3
+  * Node: cluster02 (2):
33afe3
     * pingd                           	: 1000      
33afe3
 
33afe3
 Operations:
33afe3
-  * Node: cluster02:
33afe3
+  * Node: cluster02 (2):
33afe3
     * httpd-bundle-ip-192.168.122.131: migration-threshold=1000000:
33afe3
       * (2) start
33afe3
       * (3) monitor: interval="60000ms"
33afe3
@@ -3909,7 +3934,7 @@ Operations:
33afe3
       * (9) probe
33afe3
     * ping: migration-threshold=1000000:
33afe3
       * (6) probe
33afe3
-  * Node: cluster01:
33afe3
+  * Node: cluster01 (1):
33afe3
     * Fencing: migration-threshold=1000000:
33afe3
       * (15) start
33afe3
       * (20) monitor: interval="60000ms"
33afe3
@@ -3933,11 +3958,11 @@ Operations:
33afe3
       * (1) probe
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
-  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
33afe3
-  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
33afe3
-  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
33afe3
-  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
33afe3
-  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
33afe3
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
+  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
33afe3
+  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
33afe3
+  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
33afe3
+  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
33afe3
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
33afe3
diff --git a/cts/cts-cli.in b/cts/cts-cli.in
33afe3
index d32bfb7ed1..457816afab 100755
33afe3
--- a/cts/cts-cli.in
33afe3
+++ b/cts/cts-cli.in
33afe3
@@ -420,7 +420,7 @@ function test_crm_mon() {
33afe3
     export CIB_file="$test_home/cli/crm_mon-partial.xml"
33afe3
 
33afe3
     desc="Text output of partially active resources"
33afe3
-    cmd="crm_mon -1"
33afe3
+    cmd="crm_mon -1 --show-detail"
33afe3
     test_assert $CRM_EX_OK 0
33afe3
 
33afe3
     desc="XML output of partially active resources"
33afe3
@@ -428,13 +428,13 @@ function test_crm_mon() {
33afe3
     test_assert_validate $CRM_EX_OK 0
33afe3
 
33afe3
     desc="Text output of partially active resources, with inactive resources"
33afe3
-    cmd="crm_mon -1 -r"
33afe3
+    cmd="crm_mon -1 -r --show-detail"
33afe3
     test_assert $CRM_EX_OK 0
33afe3
 
33afe3
     # XML already includes inactive resources
33afe3
 
33afe3
     desc="Complete brief text output, with inactive resources"
33afe3
-    cmd="crm_mon -1 -r --include=all --brief"
33afe3
+    cmd="crm_mon -1 -r --include=all --brief --show-detail"
33afe3
     test_assert $CRM_EX_OK 0
33afe3
 
33afe3
     # XML does not have a brief output option
33afe3
@@ -452,11 +452,11 @@ function test_crm_mon() {
33afe3
     test_assert $CRM_EX_OK 0
33afe3
 
33afe3
     desc="Text output of inactive member of partially active group"
33afe3
-    cmd="crm_mon -1 --resource=dummy-2"
33afe3
+    cmd="crm_mon -1 --resource=dummy-2 --show-detail"
33afe3
     test_assert $CRM_EX_OK 0
33afe3
 
33afe3
     desc="Complete brief text output grouped by node, with inactive resources"
33afe3
-    cmd="crm_mon -1 -r --include=all --group-by-node --brief"
33afe3
+    cmd="crm_mon -1 -r --include=all --group-by-node --brief --show-detail"
33afe3
     test_assert $CRM_EX_OK 0
33afe3
 
33afe3
     desc="Text output of partially active resources, with inactive resources, filtered by node"
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From da14053e5957d84ed0647688d37733adc2f988a3 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Mon, 29 Nov 2021 15:05:42 -0500
33afe3
Subject: [PATCH 12/21] Test: scheduler: Add tests for failed probe operations.
33afe3
33afe3
This adds identical sets of tests for primitive resources and cloned
33afe3
resources.  For the moment, the output reflects the current state of the
33afe3
code.  No changes have been made to properly handle these operations
33afe3
yet.
33afe3
33afe3
Each set has three resources, and each is set up with a slightly
33afe3
different configuration of probe failures:
33afe3
33afe3
(1) - Maskable probe failure on each node.
33afe3
(2) - Maskable probe failure on one node, successful "not running" probe
33afe3
      on the other node.  The resource should be started on the node
33afe3
      where "not running" was returned.
33afe3
(3) - Maskable probe failure on one node, non-maskable probe failure on
33afe3
      the other node.  The resource should not be running anywhere, and
33afe3
      should be stopped on the node with the non-maskable failure.
33afe3
---
33afe3
 cts/cts-scheduler.in                          |   2 +
33afe3
 cts/scheduler/dot/failed-probe-clone.dot      |  30 ++++
33afe3
 cts/scheduler/dot/failed-probe-primitive.dot  |   4 +
33afe3
 cts/scheduler/exp/failed-probe-clone.exp      | 141 ++++++++++++++++++
33afe3
 cts/scheduler/exp/failed-probe-primitive.exp  |  20 +++
33afe3
 .../scores/failed-probe-clone.scores          |  33 ++++
33afe3
 .../scores/failed-probe-primitive.scores      |   9 ++
33afe3
 .../summary/failed-probe-clone.summary        |  46 ++++++
33afe3
 .../summary/failed-probe-primitive.summary    |  27 ++++
33afe3
 cts/scheduler/xml/failed-probe-clone.xml      | 110 ++++++++++++++
33afe3
 cts/scheduler/xml/failed-probe-primitive.xml  |  71 +++++++++
33afe3
 11 files changed, 493 insertions(+)
33afe3
 create mode 100644 cts/scheduler/dot/failed-probe-clone.dot
33afe3
 create mode 100644 cts/scheduler/dot/failed-probe-primitive.dot
33afe3
 create mode 100644 cts/scheduler/exp/failed-probe-clone.exp
33afe3
 create mode 100644 cts/scheduler/exp/failed-probe-primitive.exp
33afe3
 create mode 100644 cts/scheduler/scores/failed-probe-clone.scores
33afe3
 create mode 100644 cts/scheduler/scores/failed-probe-primitive.scores
33afe3
 create mode 100644 cts/scheduler/summary/failed-probe-clone.summary
33afe3
 create mode 100644 cts/scheduler/summary/failed-probe-primitive.summary
33afe3
 create mode 100644 cts/scheduler/xml/failed-probe-clone.xml
33afe3
 create mode 100644 cts/scheduler/xml/failed-probe-primitive.xml
33afe3
33afe3
diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
33afe3
index 17fd6cefdf..3abcbc6c9d 100644
33afe3
--- a/cts/cts-scheduler.in
33afe3
+++ b/cts/cts-scheduler.in
33afe3
@@ -113,6 +113,8 @@ TESTS = [
33afe3
         [ "probe-3", "Probe (pending node)" ],
33afe3
         [ "probe-4", "Probe (pending node + stopped resource)" ],
33afe3
         [ "probe-pending-node", "Probe (pending node + unmanaged resource)" ],
33afe3
+        [ "failed-probe-primitive", "Maskable vs. unmaskable probe failures on primitive resources" ],
33afe3
+        [ "failed-probe-clone", "Maskable vs. unmaskable probe failures on cloned resources" ],
33afe3
         [ "standby", "Standby" ],
33afe3
         [ "comments", "Comments" ],
33afe3
     ],
33afe3
diff --git a/cts/scheduler/dot/failed-probe-clone.dot b/cts/scheduler/dot/failed-probe-clone.dot
33afe3
new file mode 100644
33afe3
index 0000000000..90536b46ed
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/dot/failed-probe-clone.dot
33afe3
@@ -0,0 +1,30 @@
33afe3
+ digraph "g" {
33afe3
+"ping-1_clear_failcount_0 cluster01" [ style=bold color="green" fontcolor="black"]
33afe3
+"ping-1_clear_failcount_0 cluster02" [ style=bold color="green" fontcolor="black"]
33afe3
+"ping-2-clone_running_0" [ style=bold color="green" fontcolor="orange"]
33afe3
+"ping-2-clone_start_0" -> "ping-2-clone_running_0" [ style = bold]
33afe3
+"ping-2-clone_start_0" -> "ping-2_start_0 cluster02" [ style = bold]
33afe3
+"ping-2-clone_start_0" [ style=bold color="green" fontcolor="orange"]
33afe3
+"ping-2_clear_failcount_0 cluster01" [ style=bold color="green" fontcolor="black"]
33afe3
+"ping-2_clear_failcount_0 cluster02" [ style=bold color="green" fontcolor="black"]
33afe3
+"ping-2_monitor_10000 cluster02" [ style=bold color="green" fontcolor="black"]
33afe3
+"ping-2_start_0 cluster02" -> "ping-2-clone_running_0" [ style = bold]
33afe3
+"ping-2_start_0 cluster02" -> "ping-2_monitor_10000 cluster02" [ style = bold]
33afe3
+"ping-2_start_0 cluster02" [ style=bold color="green" fontcolor="black"]
33afe3
+"ping-3-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
33afe3
+"ping-3-clone_start_0" -> "ping-3-clone_running_0" [ style = dashed]
33afe3
+"ping-3-clone_start_0" -> "ping-3_start_0 <none>" [ style = dashed]
33afe3
+"ping-3-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
33afe3
+"ping-3-clone_stop_0" -> "ping-3-clone_stopped_0" [ style = bold]
33afe3
+"ping-3-clone_stop_0" -> "ping-3_stop_0 cluster01" [ style = bold]
33afe3
+"ping-3-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
33afe3
+"ping-3-clone_stopped_0" -> "ping-3-clone_start_0" [ style = dashed]
33afe3
+"ping-3-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
33afe3
+"ping-3_clear_failcount_0 cluster01" [ style=bold color="green" fontcolor="black"]
33afe3
+"ping-3_clear_failcount_0 cluster02" [ style=bold color="green" fontcolor="black"]
33afe3
+"ping-3_start_0 <none>" -> "ping-3-clone_running_0" [ style = dashed]
33afe3
+"ping-3_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
33afe3
+"ping-3_stop_0 cluster01" -> "ping-3-clone_stopped_0" [ style = bold]
33afe3
+"ping-3_stop_0 cluster01" -> "ping-3_start_0 <none>" [ style = dashed]
33afe3
+"ping-3_stop_0 cluster01" [ style=bold color="green" fontcolor="black"]
33afe3
+}
33afe3
diff --git a/cts/scheduler/dot/failed-probe-primitive.dot b/cts/scheduler/dot/failed-probe-primitive.dot
33afe3
new file mode 100644
33afe3
index 0000000000..6e0c83216a
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/dot/failed-probe-primitive.dot
33afe3
@@ -0,0 +1,4 @@
33afe3
+ digraph "g" {
33afe3
+"dummy-2_start_0 cluster02" [ style=bold color="green" fontcolor="black"]
33afe3
+"dummy-3_stop_0 cluster01" [ style=bold color="green" fontcolor="black"]
33afe3
+}
33afe3
diff --git a/cts/scheduler/exp/failed-probe-clone.exp b/cts/scheduler/exp/failed-probe-clone.exp
33afe3
new file mode 100644
33afe3
index 0000000000..6be18935bf
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/exp/failed-probe-clone.exp
33afe3
@@ -0,0 +1,141 @@
33afe3
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
33afe3
+  <synapse id="0">
33afe3
+    <action_set>
33afe3
+      <crm_event id="6" operation="clear_failcount" operation_key="ping-1_clear_failcount_0" internal_operation_key="ping-1:0_clear_failcount_0" on_node="cluster02" on_node_uuid="2">
33afe3
+        <primitive id="ping-1" long-id="ping-1:0" class="ocf" provider="pacemaker" type="ping"/>
33afe3
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.1" multiplier="1000"/>
33afe3
+      </crm_event>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+  <synapse id="1">
33afe3
+    <action_set>
33afe3
+      <crm_event id="2" operation="clear_failcount" operation_key="ping-1_clear_failcount_0" internal_operation_key="ping-1:0_clear_failcount_0" on_node="cluster01" on_node_uuid="1">
33afe3
+        <primitive id="ping-1" long-id="ping-1:0" class="ocf" provider="pacemaker" type="ping"/>
33afe3
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.1" multiplier="1000"/>
33afe3
+      </crm_event>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+  <synapse id="2">
33afe3
+    <action_set>
33afe3
+      <rsc_op id="17" operation="monitor" operation_key="ping-2_monitor_10000" internal_operation_key="ping-2:0_monitor_10000" on_node="cluster02" on_node_uuid="2">
33afe3
+        <primitive id="ping-2" long-id="ping-2:0" class="ocf" provider="pacemaker" type="ping"/>
33afe3
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_timeout="60000"  dampen="5s" host_list="192.168.122.2" multiplier="1000"/>
33afe3
+      </rsc_op>
33afe3
+    </action_set>
33afe3
+    <inputs>
33afe3
+      <trigger>
33afe3
+        <rsc_op id="16" operation="start" operation_key="ping-2_start_0" internal_operation_key="ping-2:0_start_0" on_node="cluster02" on_node_uuid="2"/>
33afe3
+      </trigger>
33afe3
+    </inputs>
33afe3
+  </synapse>
33afe3
+  <synapse id="3">
33afe3
+    <action_set>
33afe3
+      <rsc_op id="16" operation="start" operation_key="ping-2_start_0" internal_operation_key="ping-2:0_start_0" on_node="cluster02" on_node_uuid="2">
33afe3
+        <primitive id="ping-2" long-id="ping-2:0" class="ocf" provider="pacemaker" type="ping"/>
33afe3
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_timeout="60000"  dampen="5s" host_list="192.168.122.2" multiplier="1000"/>
33afe3
+      </rsc_op>
33afe3
+    </action_set>
33afe3
+    <inputs>
33afe3
+      <trigger>
33afe3
+        <pseudo_event id="18" operation="start" operation_key="ping-2-clone_start_0"/>
33afe3
+      </trigger>
33afe3
+    </inputs>
33afe3
+  </synapse>
33afe3
+  <synapse id="4">
33afe3
+    <action_set>
33afe3
+      <crm_event id="7" operation="clear_failcount" operation_key="ping-2_clear_failcount_0" internal_operation_key="ping-2:0_clear_failcount_0" on_node="cluster02" on_node_uuid="2">
33afe3
+        <primitive id="ping-2" long-id="ping-2:0" class="ocf" provider="pacemaker" type="ping"/>
33afe3
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.2" multiplier="1000"/>
33afe3
+      </crm_event>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+  <synapse id="5">
33afe3
+    <action_set>
33afe3
+      <crm_event id="3" operation="clear_failcount" operation_key="ping-2_clear_failcount_0" internal_operation_key="ping-2:0_clear_failcount_0" on_node="cluster01" on_node_uuid="1">
33afe3
+        <primitive id="ping-2" long-id="ping-2:0" class="ocf" provider="pacemaker" type="ping"/>
33afe3
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.2" multiplier="1000"/>
33afe3
+      </crm_event>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+  <synapse id="6" priority="1000000">
33afe3
+    <action_set>
33afe3
+      <pseudo_event id="19" operation="running" operation_key="ping-2-clone_running_0">
33afe3
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
33afe3
+      </pseudo_event>
33afe3
+    </action_set>
33afe3
+    <inputs>
33afe3
+      <trigger>
33afe3
+        <rsc_op id="16" operation="start" operation_key="ping-2_start_0" internal_operation_key="ping-2:0_start_0" on_node="cluster02" on_node_uuid="2"/>
33afe3
+      </trigger>
33afe3
+      <trigger>
33afe3
+        <pseudo_event id="18" operation="start" operation_key="ping-2-clone_start_0"/>
33afe3
+      </trigger>
33afe3
+    </inputs>
33afe3
+  </synapse>
33afe3
+  <synapse id="7">
33afe3
+    <action_set>
33afe3
+      <pseudo_event id="18" operation="start" operation_key="ping-2-clone_start_0">
33afe3
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
33afe3
+      </pseudo_event>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+  <synapse id="8">
33afe3
+    <action_set>
33afe3
+      <rsc_op id="5" operation="stop" operation_key="ping-3_stop_0" internal_operation_key="ping-3:0_stop_0" on_node="cluster01" on_node_uuid="1">
33afe3
+        <primitive id="ping-3" long-id="ping-3:0" class="ocf" provider="pacemaker" type="ping"/>
33afe3
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.3" multiplier="1000"/>
33afe3
+      </rsc_op>
33afe3
+    </action_set>
33afe3
+    <inputs>
33afe3
+      <trigger>
33afe3
+        <pseudo_event id="24" operation="stop" operation_key="ping-3-clone_stop_0"/>
33afe3
+      </trigger>
33afe3
+    </inputs>
33afe3
+  </synapse>
33afe3
+  <synapse id="9">
33afe3
+    <action_set>
33afe3
+      <crm_event id="4" operation="clear_failcount" operation_key="ping-3_clear_failcount_0" internal_operation_key="ping-3:0_clear_failcount_0" on_node="cluster01" on_node_uuid="1">
33afe3
+        <primitive id="ping-3" long-id="ping-3:0" class="ocf" provider="pacemaker" type="ping"/>
33afe3
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.3" multiplier="1000"/>
33afe3
+      </crm_event>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+  <synapse id="10">
33afe3
+    <action_set>
33afe3
+      <crm_event id="8" operation="clear_failcount" operation_key="ping-3_clear_failcount_0" internal_operation_key="ping-3:1_clear_failcount_0" on_node="cluster02" on_node_uuid="2">
33afe3
+        <primitive id="ping-3" long-id="ping-3:1" class="ocf" provider="pacemaker" type="ping"/>
33afe3
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.3" multiplier="1000"/>
33afe3
+      </crm_event>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+  <synapse id="11" priority="1000000">
33afe3
+    <action_set>
33afe3
+      <pseudo_event id="25" operation="stopped" operation_key="ping-3-clone_stopped_0">
33afe3
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
33afe3
+      </pseudo_event>
33afe3
+    </action_set>
33afe3
+    <inputs>
33afe3
+      <trigger>
33afe3
+        <rsc_op id="5" operation="stop" operation_key="ping-3_stop_0" internal_operation_key="ping-3:0_stop_0" on_node="cluster01" on_node_uuid="1"/>
33afe3
+      </trigger>
33afe3
+      <trigger>
33afe3
+        <pseudo_event id="24" operation="stop" operation_key="ping-3-clone_stop_0"/>
33afe3
+      </trigger>
33afe3
+    </inputs>
33afe3
+  </synapse>
33afe3
+  <synapse id="12">
33afe3
+    <action_set>
33afe3
+      <pseudo_event id="24" operation="stop" operation_key="ping-3-clone_stop_0">
33afe3
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
33afe3
+      </pseudo_event>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+</transition_graph>
33afe3
diff --git a/cts/scheduler/exp/failed-probe-primitive.exp b/cts/scheduler/exp/failed-probe-primitive.exp
33afe3
new file mode 100644
33afe3
index 0000000000..d0d8aa44dc
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/exp/failed-probe-primitive.exp
33afe3
@@ -0,0 +1,20 @@
33afe3
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
33afe3
+  <synapse id="0">
33afe3
+    <action_set>
33afe3
+      <rsc_op id="5" operation="start" operation_key="dummy-2_start_0" on_node="cluster02" on_node_uuid="2">
33afe3
+        <primitive id="dummy-2" class="ocf" provider="pacemaker" type="Dummy"/>
33afe3
+        <attributes CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" />
33afe3
+      </rsc_op>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+  <synapse id="1">
33afe3
+    <action_set>
33afe3
+      <rsc_op id="2" operation="stop" operation_key="dummy-3_stop_0" on_node="cluster01" on_node_uuid="1">
33afe3
+        <primitive id="dummy-3" class="ocf" provider="pacemaker" type="Dummy"/>
33afe3
+        <attributes CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" />
33afe3
+      </rsc_op>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+</transition_graph>
33afe3
diff --git a/cts/scheduler/scores/failed-probe-clone.scores b/cts/scheduler/scores/failed-probe-clone.scores
33afe3
new file mode 100644
33afe3
index 0000000000..7418b7f153
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/scores/failed-probe-clone.scores
33afe3
@@ -0,0 +1,33 @@
33afe3
+
33afe3
+pcmk__clone_allocate: ping-1-clone allocation score on cluster01: -INFINITY
33afe3
+pcmk__clone_allocate: ping-1-clone allocation score on cluster02: -INFINITY
33afe3
+pcmk__clone_allocate: ping-1:0 allocation score on cluster01: -INFINITY
33afe3
+pcmk__clone_allocate: ping-1:0 allocation score on cluster02: -INFINITY
33afe3
+pcmk__clone_allocate: ping-1:1 allocation score on cluster01: -INFINITY
33afe3
+pcmk__clone_allocate: ping-1:1 allocation score on cluster02: -INFINITY
33afe3
+pcmk__clone_allocate: ping-2-clone allocation score on cluster01: -INFINITY
33afe3
+pcmk__clone_allocate: ping-2-clone allocation score on cluster02: 0
33afe3
+pcmk__clone_allocate: ping-2:0 allocation score on cluster01: -INFINITY
33afe3
+pcmk__clone_allocate: ping-2:0 allocation score on cluster02: 0
33afe3
+pcmk__clone_allocate: ping-2:1 allocation score on cluster01: -INFINITY
33afe3
+pcmk__clone_allocate: ping-2:1 allocation score on cluster02: 0
33afe3
+pcmk__clone_allocate: ping-3-clone allocation score on cluster01: -INFINITY
33afe3
+pcmk__clone_allocate: ping-3-clone allocation score on cluster02: -INFINITY
33afe3
+pcmk__clone_allocate: ping-3:0 allocation score on cluster01: -INFINITY
33afe3
+pcmk__clone_allocate: ping-3:0 allocation score on cluster02: -INFINITY
33afe3
+pcmk__clone_allocate: ping-3:1 allocation score on cluster01: -INFINITY
33afe3
+pcmk__clone_allocate: ping-3:1 allocation score on cluster02: -INFINITY
33afe3
+pcmk__native_allocate: Fencing allocation score on cluster01: 0
33afe3
+pcmk__native_allocate: Fencing allocation score on cluster02: 0
33afe3
+pcmk__native_allocate: ping-1:0 allocation score on cluster01: -INFINITY
33afe3
+pcmk__native_allocate: ping-1:0 allocation score on cluster02: -INFINITY
33afe3
+pcmk__native_allocate: ping-1:1 allocation score on cluster01: -INFINITY
33afe3
+pcmk__native_allocate: ping-1:1 allocation score on cluster02: -INFINITY
33afe3
+pcmk__native_allocate: ping-2:0 allocation score on cluster01: -INFINITY
33afe3
+pcmk__native_allocate: ping-2:0 allocation score on cluster02: 0
33afe3
+pcmk__native_allocate: ping-2:1 allocation score on cluster01: -INFINITY
33afe3
+pcmk__native_allocate: ping-2:1 allocation score on cluster02: -INFINITY
33afe3
+pcmk__native_allocate: ping-3:0 allocation score on cluster01: -INFINITY
33afe3
+pcmk__native_allocate: ping-3:0 allocation score on cluster02: -INFINITY
33afe3
+pcmk__native_allocate: ping-3:1 allocation score on cluster01: -INFINITY
33afe3
+pcmk__native_allocate: ping-3:1 allocation score on cluster02: -INFINITY
33afe3
diff --git a/cts/scheduler/scores/failed-probe-primitive.scores b/cts/scheduler/scores/failed-probe-primitive.scores
33afe3
new file mode 100644
33afe3
index 0000000000..f313029451
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/scores/failed-probe-primitive.scores
33afe3
@@ -0,0 +1,9 @@
33afe3
+
33afe3
+pcmk__native_allocate: Fencing allocation score on cluster01: 0
33afe3
+pcmk__native_allocate: Fencing allocation score on cluster02: 0
33afe3
+pcmk__native_allocate: dummy-1 allocation score on cluster01: -INFINITY
33afe3
+pcmk__native_allocate: dummy-1 allocation score on cluster02: -INFINITY
33afe3
+pcmk__native_allocate: dummy-2 allocation score on cluster01: -INFINITY
33afe3
+pcmk__native_allocate: dummy-2 allocation score on cluster02: 0
33afe3
+pcmk__native_allocate: dummy-3 allocation score on cluster01: -INFINITY
33afe3
+pcmk__native_allocate: dummy-3 allocation score on cluster02: -INFINITY
33afe3
diff --git a/cts/scheduler/summary/failed-probe-clone.summary b/cts/scheduler/summary/failed-probe-clone.summary
33afe3
new file mode 100644
33afe3
index 0000000000..ca15c302aa
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/summary/failed-probe-clone.summary
33afe3
@@ -0,0 +1,46 @@
33afe3
+Current cluster status:
33afe3
+  * Node List:
33afe3
+    * Online: [ cluster01 cluster02 ]
33afe3
+
33afe3
+  * Full List of Resources:
33afe3
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
+    * Clone Set: ping-1-clone [ping-1]:
33afe3
+      * Stopped: [ cluster01 cluster02 ]
33afe3
+    * Clone Set: ping-2-clone [ping-2]:
33afe3
+      * Stopped: [ cluster01 cluster02 ]
33afe3
+    * Clone Set: ping-3-clone [ping-3]:
33afe3
+      * ping-3	(ocf:pacemaker:ping):	 FAILED cluster01
33afe3
+      * Stopped: [ cluster02 ]
33afe3
+
33afe3
+Transition Summary:
33afe3
+  * Start      ping-2:0     ( cluster02 )
33afe3
+  * Stop       ping-3:0     ( cluster01 )  due to node availability
33afe3
+
33afe3
+Executing Cluster Transition:
33afe3
+  * Cluster action:  clear_failcount for ping-1 on cluster02
33afe3
+  * Cluster action:  clear_failcount for ping-1 on cluster01
33afe3
+  * Cluster action:  clear_failcount for ping-2 on cluster02
33afe3
+  * Cluster action:  clear_failcount for ping-2 on cluster01
33afe3
+  * Pseudo action:   ping-2-clone_start_0
33afe3
+  * Cluster action:  clear_failcount for ping-3 on cluster01
33afe3
+  * Cluster action:  clear_failcount for ping-3 on cluster02
33afe3
+  * Pseudo action:   ping-3-clone_stop_0
33afe3
+  * Resource action: ping-2          start on cluster02
33afe3
+  * Pseudo action:   ping-2-clone_running_0
33afe3
+  * Resource action: ping-3          stop on cluster01
33afe3
+  * Pseudo action:   ping-3-clone_stopped_0
33afe3
+  * Resource action: ping-2          monitor=10000 on cluster02
33afe3
+
33afe3
+Revised Cluster Status:
33afe3
+  * Node List:
33afe3
+    * Online: [ cluster01 cluster02 ]
33afe3
+
33afe3
+  * Full List of Resources:
33afe3
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
+    * Clone Set: ping-1-clone [ping-1]:
33afe3
+      * Stopped: [ cluster01 cluster02 ]
33afe3
+    * Clone Set: ping-2-clone [ping-2]:
33afe3
+      * Started: [ cluster02 ]
33afe3
+      * Stopped: [ cluster01 ]
33afe3
+    * Clone Set: ping-3-clone [ping-3]:
33afe3
+      * Stopped: [ cluster01 cluster02 ]
33afe3
diff --git a/cts/scheduler/summary/failed-probe-primitive.summary b/cts/scheduler/summary/failed-probe-primitive.summary
33afe3
new file mode 100644
33afe3
index 0000000000..a634e7f00b
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/summary/failed-probe-primitive.summary
33afe3
@@ -0,0 +1,27 @@
33afe3
+Current cluster status:
33afe3
+  * Node List:
33afe3
+    * Online: [ cluster01 cluster02 ]
33afe3
+
33afe3
+  * Full List of Resources:
33afe3
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
+    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped
33afe3
+    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped
33afe3
+    * dummy-3	(ocf:pacemaker:Dummy):	 FAILED cluster01
33afe3
+
33afe3
+Transition Summary:
33afe3
+  * Start      dummy-2     ( cluster02 )
33afe3
+  * Stop       dummy-3     ( cluster01 )  due to node availability
33afe3
+
33afe3
+Executing Cluster Transition:
33afe3
+  * Resource action: dummy-2         start on cluster02
33afe3
+  * Resource action: dummy-3         stop on cluster01
33afe3
+
33afe3
+Revised Cluster Status:
33afe3
+  * Node List:
33afe3
+    * Online: [ cluster01 cluster02 ]
33afe3
+
33afe3
+  * Full List of Resources:
33afe3
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
+    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped
33afe3
+    * dummy-2	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
+    * dummy-3	(ocf:pacemaker:Dummy):	 Stopped
33afe3
diff --git a/cts/scheduler/xml/failed-probe-clone.xml b/cts/scheduler/xml/failed-probe-clone.xml
33afe3
new file mode 100644
33afe3
index 0000000000..f677585bab
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/xml/failed-probe-clone.xml
33afe3
@@ -0,0 +1,110 @@
33afe3
+<cib crm_feature_set="3.3.0" validate-with="pacemaker-3.3" epoch="1" num_updates="37" admin_epoch="1" cib-last-written="Tue May  5 12:04:36 2020" update-origin="cluster01" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2">
33afe3
+  <configuration>
33afe3
+    <crm_config>
33afe3
+      <cluster_property_set id="cib-bootstrap-options">
33afe3
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
33afe3
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-1.e97f9675f.git.el7-e97f9675f"/>
33afe3
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
33afe3
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="test-cluster"/>
33afe3
+        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
33afe3
+        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
33afe3
+      </cluster_property_set>
33afe3
+    </crm_config>
33afe3
+    <nodes>
33afe3
+      <node id="1" uname="cluster01"/>
33afe3
+      <node id="2" uname="cluster02"/>
33afe3
+    </nodes>
33afe3
+    <resources>
33afe3
+      <primitive class="stonith" id="Fencing" type="fence_xvm">
33afe3
+        <instance_attributes id="Fencing-instance_attributes">
33afe3
+          <nvpair id="Fencing-instance_attributes-ip_family" name="ip_family" value="ipv4"/>
33afe3
+        </instance_attributes>
33afe3
+        <operations>
33afe3
+          <op id="Fencing-monitor-interval-60s" interval="60s" name="monitor"/>
33afe3
+        </operations>
33afe3
+      </primitive>
33afe3
+      <clone id="ping-1-clone">
33afe3
+        <primitive class="ocf" id="ping-1" provider="pacemaker" type="ping">
33afe3
+          <instance_attributes id="ping-1-instance_attributes">
33afe3
+            <nvpair id="ping-1-instance_attributes-dampen" name="dampen" value="5s"/>
33afe3
+            <nvpair id="ping-1-instance_attributes-host_list" name="host_list" value="192.168.122.1"/>
33afe3
+            <nvpair id="ping-1-instance_attributes-multiplier" name="multiplier" value="1000"/>
33afe3
+          </instance_attributes>
33afe3
+          <operations>
33afe3
+            <op id="ping-1-monitor-interval-10s" interval="10s" name="monitor" timeout="60s"/>
33afe3
+            <op id="ping-1-start-interval-0s" interval="0s" name="start" timeout="60s"/>
33afe3
+            <op id="ping-1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
33afe3
+          </operations>
33afe3
+        </primitive>
33afe3
+      </clone>
33afe3
+      <clone id="ping-2-clone">
33afe3
+        <primitive class="ocf" id="ping-2" provider="pacemaker" type="ping">
33afe3
+          <instance_attributes id="ping-2-instance_attributes">
33afe3
+            <nvpair id="ping-2-instance_attributes-dampen" name="dampen" value="5s"/>
33afe3
+            <nvpair id="ping-2-instance_attributes-host_list" name="host_list" value="192.168.122.2"/>
33afe3
+            <nvpair id="ping-2-instance_attributes-multiplier" name="multiplier" value="1000"/>
33afe3
+          </instance_attributes>
33afe3
+          <operations>
33afe3
+            <op id="ping-2-monitor-interval-10s" interval="10s" name="monitor" timeout="60s"/>
33afe3
+            <op id="ping-2-start-interval-0s" interval="0s" name="start" timeout="60s"/>
33afe3
+            <op id="ping-2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
33afe3
+          </operations>
33afe3
+        </primitive>
33afe3
+      </clone>
33afe3
+      <clone id="ping-3-clone">
33afe3
+        <primitive class="ocf" id="ping-3" provider="pacemaker" type="ping">
33afe3
+          <instance_attributes id="ping-3-instance_attributes">
33afe3
+            <nvpair id="ping-3-instance_attributes-dampen" name="dampen" value="5s"/>
33afe3
+            <nvpair id="ping-3-instance_attributes-host_list" name="host_list" value="192.168.122.3"/>
33afe3
+            <nvpair id="ping-3-instance_attributes-multiplier" name="multiplier" value="1000"/>
33afe3
+          </instance_attributes>
33afe3
+          <operations>
33afe3
+            <op id="ping-3-monitor-interval-10s" interval="10s" name="monitor" timeout="60s"/>
33afe3
+            <op id="ping-3-start-interval-0s" interval="0s" name="start" timeout="60s"/>
33afe3
+            <op id="ping-3-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
33afe3
+          </operations>
33afe3
+        </primitive>
33afe3
+      </clone>
33afe3
+    </resources>
33afe3
+    <constraints/>
33afe3
+  </configuration>
33afe3
+  <status>
33afe3
+    <node_state id="1" uname="cluster01" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
33afe3
+      <lrm id="1">
33afe3
+        <lrm_resources>
33afe3
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
33afe3
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="3" rc-code="0" op-status="0" interval="0" last-rc-change="1588951272" exec-time="36" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
33afe3
+            <lrm_rsc_op id="Fencing_monitor_60000" operation_key="Fencing_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="4" rc-code="0" op-status="0" interval="60000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="a88218bb6c7dc47e6586fc75fc2a8d69"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="ping-1" class="ocf" provider="pacemaker" type="ping">
33afe3
+            <lrm_rsc_op id="ping-1_last_failure_0" operation_key="ping-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="5" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="ping-2" class="ocf" provider="pacemaker" type="ping">
33afe3
+            <lrm_rsc_op id="ping-2_last_failure_0" operation_key="ping-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="6" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="ping-3" class="ocf" provider="pacemaker" type="ping">
33afe3
+            <lrm_rsc_op id="ping-3_last_failure_0" operation_key="ping-3_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="9:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:4;9:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="9" rc-code="4" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+        </lrm_resources>
33afe3
+      </lrm>
33afe3
+    </node_state>
33afe3
+    <node_state id="2" uname="cluster02" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
33afe3
+      <lrm id="2">
33afe3
+        <lrm_resources>
33afe3
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
33afe3
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:7;1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1588951263" exec-time="3" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="ping-1" class="ocf" provider="pacemaker" type="ping">
33afe3
+            <lrm_rsc_op id="ping-1_last_failure_0" operation_key="ping-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="ping-2" class="ocf" provider="pacemaker" type="ping">
33afe3
+            <lrm_rsc_op id="ping-2_last_failure_0" operation_key="ping-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="7:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;7:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="7" rc-code="7" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="ping-3" class="ocf" provider="pacemaker" type="ping">
33afe3
+            <lrm_rsc_op id="ping-3_last_failure_0" operation_key="ping-3_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="8:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;8:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="8" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+        </lrm_resources>
33afe3
+      </lrm>
33afe3
+    </node_state>
33afe3
+  </status>
33afe3
+</cib>
33afe3
diff --git a/cts/scheduler/xml/failed-probe-primitive.xml b/cts/scheduler/xml/failed-probe-primitive.xml
33afe3
new file mode 100644
33afe3
index 0000000000..0c2f6416f5
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/xml/failed-probe-primitive.xml
33afe3
@@ -0,0 +1,71 @@
33afe3
+<cib crm_feature_set="3.3.0" validate-with="pacemaker-3.3" epoch="1" num_updates="37" admin_epoch="1" cib-last-written="Tue May  5 12:04:36 2020" update-origin="cluster01" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2">
33afe3
+  <configuration>
33afe3
+    <crm_config>
33afe3
+      <cluster_property_set id="cib-bootstrap-options">
33afe3
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
33afe3
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-1.e97f9675f.git.el7-e97f9675f"/>
33afe3
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
33afe3
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="test-cluster"/>
33afe3
+        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
33afe3
+        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
33afe3
+      </cluster_property_set>
33afe3
+    </crm_config>
33afe3
+    <nodes>
33afe3
+      <node id="1" uname="cluster01"/>
33afe3
+      <node id="2" uname="cluster02"/>
33afe3
+    </nodes>
33afe3
+    <resources>
33afe3
+      <primitive class="stonith" id="Fencing" type="fence_xvm">
33afe3
+        <instance_attributes id="Fencing-instance_attributes">
33afe3
+          <nvpair id="Fencing-instance_attributes-ip_family" name="ip_family" value="ipv4"/>
33afe3
+        </instance_attributes>
33afe3
+        <operations>
33afe3
+          <op id="Fencing-monitor-interval-60s" interval="60s" name="monitor"/>
33afe3
+        </operations>
33afe3
+      </primitive>
33afe3
+      <primitive class="ocf" id="dummy-1" provider="pacemaker" type="Dummy"/>
33afe3
+      <primitive class="ocf" id="dummy-2" provider="pacemaker" type="Dummy"/>
33afe3
+      <primitive class="ocf" id="dummy-3" provider="pacemaker" type="Dummy"/>
33afe3
+    </resources>
33afe3
+    <constraints/>
33afe3
+  </configuration>
33afe3
+  <status>
33afe3
+    <node_state id="1" uname="cluster01" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
33afe3
+      <lrm id="1">
33afe3
+        <lrm_resources>
33afe3
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
33afe3
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="3" rc-code="0" op-status="0" interval="0" last-rc-change="1588951272" exec-time="36" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
33afe3
+            <lrm_rsc_op id="Fencing_monitor_60000" operation_key="Fencing_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="4" rc-code="0" op-status="0" interval="60000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="a88218bb6c7dc47e6586fc75fc2a8d69"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-1_last_failure_0" operation_key="dummy-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="5" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="6" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="dummy-3" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-3_last_failure_0" operation_key="dummy-3_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="9:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:4;9:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="9" rc-code="4" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+        </lrm_resources>
33afe3
+      </lrm>
33afe3
+    </node_state>
33afe3
+    <node_state id="2" uname="cluster02" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
33afe3
+      <lrm id="2">
33afe3
+        <lrm_resources>
33afe3
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
33afe3
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:7;1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1588951263" exec-time="3" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-1_last_failure_0" operation_key="dummy-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="7:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;7:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="7" rc-code="7" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="dummy-3" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-3_last_failure_0" operation_key="dummy-3_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="8:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;8:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="8" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+        </lrm_resources>
33afe3
+      </lrm>
33afe3
+    </node_state>
33afe3
+  </status>
33afe3
+</cib>
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 271d50e7d6b0ee5ef670b571c6d7aae9272b75ad Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Thu, 11 Nov 2021 13:57:05 -0500
33afe3
Subject: [PATCH 13/21] Feature: scheduler: Don't output failed resource
33afe3
 probes...
33afe3
33afe3
in the crm_mon "Failed Resource Actions" section.  It is expected that
33afe3
these one-off probes will fail, in which case displaying them in that
33afe3
section can just come across as confusing to the user.
33afe3
33afe3
And update the crm_mon test output to account for these changes.
33afe3
33afe3
See: rhbz#1506372
33afe3
---
33afe3
 cts/cli/regression.crm_mon.exp | 20 --------------------
33afe3
 lib/pengine/pe_output.c        |  4 ++++
33afe3
 2 files changed, 4 insertions(+), 20 deletions(-)
33afe3
33afe3
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
33afe3
index d7b9d98e2c..b1643f8b29 100644
33afe3
--- a/cts/cli/regression.crm_mon.exp
33afe3
+++ b/cts/cli/regression.crm_mon.exp
33afe3
@@ -3498,10 +3498,6 @@ Active Resources:
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
-  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
33afe3
-  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
33afe3
-  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
33afe3
-  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
33afe3
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources
33afe3
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
33afe3
@@ -3646,10 +3642,6 @@ Failed Resource Actions:
33afe3
   </node_history>
33afe3
   <failures>
33afe3
     <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
-    <failure op_key="dummy-4_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="0" interval="0" task="monitor"/>
33afe3
-    <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
-    <failure op_key="ping_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="6" status="complete" last-rc-change="2021-11-18 13:11:42 -05:00" queued="0" exec="0" interval="0" task="monitor"/>
33afe3
-    <failure op_key="httpd_monitor_0" node="httpd-bundle-1" exitstatus="invalid parameter" exitreason="" exitcode="2" call="1" status="complete" last-rc-change="2020-05-27 15:43:09 -04:00" queued="0" exec="0" interval="0" task="monitor"/>
33afe3
   </failures>
33afe3
   <status code="0" message="OK"/>
33afe3
 </pacemaker-result>
33afe3
@@ -3693,10 +3685,6 @@ Full List of Resources:
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
-  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
33afe3
-  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
33afe3
-  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
33afe3
-  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
33afe3
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
33afe3
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
33afe3
@@ -3784,10 +3772,6 @@ Operations:
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
-  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
33afe3
-  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
33afe3
-  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
33afe3
-  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
33afe3
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
33afe3
@@ -3959,10 +3943,6 @@ Operations:
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
-  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
33afe3
-  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
33afe3
-  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
33afe3
-  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
33afe3
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
33afe3
diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
33afe3
index 715e001d51..84684598dd 100644
33afe3
--- a/lib/pengine/pe_output.c
33afe3
+++ b/lib/pengine/pe_output.c
33afe3
@@ -1370,6 +1370,10 @@ failed_action_list(pcmk__output_t *out, va_list args) {
33afe3
             continue;
33afe3
         }
33afe3
 
33afe3
+        if (pcmk_xe_mask_probe_failure(xml_op)) {
33afe3
+            continue;
33afe3
+        }
33afe3
+
33afe3
         id = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
33afe3
         if (parse_op_key(id ? id : ID(xml_op), &rsc, NULL, NULL) == FALSE) {
33afe3
             continue;
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 90f641b9223c64701d494297ce3dd3382365acb8 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Tue, 9 Nov 2021 10:11:19 -0500
33afe3
Subject: [PATCH 14/21] Feature: scheduler: Add a function for finding a failed
33afe3
 probe action...
33afe3
33afe3
for a given resource ID.  Optionally, a node ID can also be given to
33afe3
restrict the failed probe action to one run on the given node.
33afe3
Otherwise, just the first failed probe action for the resource ID will
33afe3
be returned.
33afe3
33afe3
See: rhbz#1506372
33afe3
---
33afe3
 include/crm/pengine/internal.h |  2 ++
33afe3
 lib/pengine/utils.c            | 42 ++++++++++++++++++++++++++++++++++
33afe3
 2 files changed, 44 insertions(+)
33afe3
33afe3
diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
33afe3
index 8c8fbaca90..58dd2e8727 100644
33afe3
--- a/include/crm/pengine/internal.h
33afe3
+++ b/include/crm/pengine/internal.h
33afe3
@@ -574,4 +574,6 @@ gboolean pe__clone_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean che
33afe3
 gboolean pe__group_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent);
33afe3
 gboolean pe__native_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent);
33afe3
 
33afe3
+xmlNode *pe__failed_probe_for_rsc(pe_resource_t *rsc, const char *name);
33afe3
+
33afe3
 #endif
33afe3
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
33afe3
index 07753e173a..3151f0120b 100644
33afe3
--- a/lib/pengine/utils.c
33afe3
+++ b/lib/pengine/utils.c
33afe3
@@ -2569,3 +2569,45 @@ pe__build_rsc_list(pe_working_set_t *data_set, const char *s) {
33afe3
 
33afe3
     return resources;
33afe3
 }
33afe3
+
33afe3
+xmlNode *
33afe3
+pe__failed_probe_for_rsc(pe_resource_t *rsc, const char *name)
33afe3
+{
33afe3
+    const char *rsc_id = rsc->id;
33afe3
+
33afe3
+    for (xmlNode *xml_op = pcmk__xml_first_child(rsc->cluster->failed); xml_op != NULL;
33afe3
+         xml_op = pcmk__xml_next(xml_op)) {
33afe3
+        const char *value = NULL;
33afe3
+        char *op_id = NULL;
33afe3
+
33afe3
+        /* This resource operation is not a failed probe. */
33afe3
+        if (!pcmk_xe_mask_probe_failure(xml_op)) {
33afe3
+            continue;
33afe3
+        }
33afe3
+
33afe3
+        /* This resource operation was not run on the given node.  Note that if name is
33afe3
+         * NULL, this will always succeed.
33afe3
+         */
33afe3
+        value = crm_element_value(xml_op, XML_LRM_ATTR_TARGET);
33afe3
+        if (value == NULL || !pcmk__str_eq(value, name, pcmk__str_casei|pcmk__str_null_matches)) {
33afe3
+            continue;
33afe3
+        }
33afe3
+
33afe3
+        /* This resource operation has no operation_key. */
33afe3
+        value = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
33afe3
+        if (!parse_op_key(value ? value : ID(xml_op), &op_id, NULL, NULL)) {
33afe3
+            continue;
33afe3
+        }
33afe3
+
33afe3
+        /* This resource operation's ID does not match the rsc_id we are looking for. */
33afe3
+        if (!pcmk__str_eq(op_id, rsc_id, pcmk__str_none)) {
33afe3
+            free(op_id);
33afe3
+            continue;
33afe3
+        }
33afe3
+
33afe3
+        free(op_id);
33afe3
+        return xml_op;
33afe3
+    }
33afe3
+
33afe3
+    return NULL;
33afe3
+}
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 2ad9774fe994554243078b131799fed0d1a6dffd Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Tue, 9 Nov 2021 15:43:24 -0500
33afe3
Subject: [PATCH 15/21] Feature: scheduler: Display the reason why a native rsc
33afe3
 probe failed.
33afe3
33afe3
If inactive resources are being shown, add an extra blurb of text to any
33afe3
stopped resources that have a failed probe action indicating why the
33afe3
probe failed.
33afe3
33afe3
And then add a new primitive resource to crm_mon-partial.xml with a
33afe3
failed probe operation and update the expected test output.
33afe3
33afe3
See: rhbz#1506372
33afe3
---
33afe3
 cts/cli/regression.crm_mon.exp                        | 10 +++++-----
33afe3
 cts/scheduler/summary/failed-probe-primitive.summary  |  8 ++++----
33afe3
 cts/scheduler/summary/multiply-active-stonith.summary |  2 +-
33afe3
 lib/pengine/native.c                                  | 11 +++++++++++
33afe3
 4 files changed, 21 insertions(+), 10 deletions(-)
33afe3
33afe3
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
33afe3
index b1643f8b29..4333caa11c 100644
33afe3
--- a/cts/cli/regression.crm_mon.exp
33afe3
+++ b/cts/cli/regression.crm_mon.exp
33afe3
@@ -3680,8 +3680,8 @@ Full List of Resources:
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
     * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
33afe3
-    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped
33afe3
-  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
+    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
33afe3
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped (not installed) 
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
   * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
@@ -3811,7 +3811,7 @@ Full List of Resources:
33afe3
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
     * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
33afe3
-    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped
33afe3
+    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
33afe3
 =#=#=#= End test: Text output of partially active group, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active group, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of active member of partially active group =#=#=#=
33afe3
@@ -3889,7 +3889,7 @@ Inactive Resources:
33afe3
     * ping	(ocf:pacemaker:ping):	 Stopped
33afe3
   * Resource Group: partially-active-group:
33afe3
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
-  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped (not installed) 
33afe3
 
33afe3
 Node Attributes:
33afe3
   * Node: cluster01 (1):
33afe3
@@ -3963,7 +3963,7 @@ Full List of Resources:
33afe3
   * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
33afe3
-  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
33afe3
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped (not installed) 
33afe3
 =#=#=#= End test: Text output of partially active resources, with inactive resources, filtered by node - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources, filtered by node
33afe3
 =#=#=#= Begin test: Text output of partially active resources, filtered by node =#=#=#=
33afe3
diff --git a/cts/scheduler/summary/failed-probe-primitive.summary b/cts/scheduler/summary/failed-probe-primitive.summary
33afe3
index a634e7f00b..ea8edae494 100644
33afe3
--- a/cts/scheduler/summary/failed-probe-primitive.summary
33afe3
+++ b/cts/scheduler/summary/failed-probe-primitive.summary
33afe3
@@ -4,8 +4,8 @@ Current cluster status:
33afe3
 
33afe3
   * Full List of Resources:
33afe3
     * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
-    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped
33afe3
-    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped
33afe3
+    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
33afe3
+    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
33afe3
     * dummy-3	(ocf:pacemaker:Dummy):	 FAILED cluster01
33afe3
 
33afe3
 Transition Summary:
33afe3
@@ -22,6 +22,6 @@ Revised Cluster Status:
33afe3
 
33afe3
   * Full List of Resources:
33afe3
     * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
-    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped
33afe3
+    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
-    * dummy-3	(ocf:pacemaker:Dummy):	 Stopped
33afe3
+    * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
33afe3
diff --git a/cts/scheduler/summary/multiply-active-stonith.summary b/cts/scheduler/summary/multiply-active-stonith.summary
33afe3
index 8ce21d68ee..ec37de03b0 100644
33afe3
--- a/cts/scheduler/summary/multiply-active-stonith.summary
33afe3
+++ b/cts/scheduler/summary/multiply-active-stonith.summary
33afe3
@@ -25,4 +25,4 @@ Revised Cluster Status:
33afe3
 
33afe3
   * Full List of Resources:
33afe3
     * fencer	(stonith:fence_ipmilan):	 Started node3
33afe3
-    * rsc1	(lsb:rsc1):	 Stopped
33afe3
+    * rsc1	(lsb:rsc1):	 Stopped (not installed) 
33afe3
diff --git a/lib/pengine/native.c b/lib/pengine/native.c
33afe3
index 36121c527f..a95c90c09a 100644
33afe3
--- a/lib/pengine/native.c
33afe3
+++ b/lib/pengine/native.c
33afe3
@@ -599,6 +599,17 @@ pcmk__native_output_string(pe_resource_t *rsc, const char *name, pe_node_t *node
33afe3
         g_string_append_printf(outstr, " %s", node->details->uname);
33afe3
     }
33afe3
 
33afe3
+    // Failed probe operation
33afe3
+    if (native_displayable_role(rsc) == RSC_ROLE_STOPPED) {
33afe3
+        xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node ? node->details->uname : NULL);
33afe3
+        if (probe_op != NULL) {
33afe3
+            int rc;
33afe3
+
33afe3
+            pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
33afe3
+            g_string_append_printf(outstr, " (%s) ", services_ocf_exitcode_str(rc));
33afe3
+        }
33afe3
+    }
33afe3
+
33afe3
     // Flags, as: (<flag> [...])
33afe3
     if (node && !(node->details->online) && node->details->unclean) {
33afe3
         have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From b9ca2e834ee01b35c03f153438ef8828b609fb38 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Thu, 18 Nov 2021 10:41:42 -0500
33afe3
Subject: [PATCH 16/21] Refactor: scheduler: Rearrange pe__clone_default.
33afe3
33afe3
Instead of the single stopped list, maintain a hash table where the keys
33afe3
are nodes and the values are the status of the node.  For now, this is
33afe3
just "Stopped" or "Stopped (disabled)" but in the future will be
33afe3
expanded to cover failed probe operations.
33afe3
---
33afe3
 lib/pengine/clone.c | 103 +++++++++++++++++++++++++++++++++++---------
33afe3
 1 file changed, 82 insertions(+), 21 deletions(-)
33afe3
33afe3
diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
33afe3
index 5569c6b6e9..58fb24d24e 100644
33afe3
--- a/lib/pengine/clone.c
33afe3
+++ b/lib/pengine/clone.c
33afe3
@@ -28,6 +28,55 @@
33afe3
 #define UNPROMOTED_INSTANCES RSC_ROLE_UNPROMOTED_S
33afe3
 #endif
33afe3
 
33afe3
+static GList *
33afe3
+sorted_hash_table_values(GHashTable *table)
33afe3
+{
33afe3
+    GList *retval = NULL;
33afe3
+    GHashTableIter iter;
33afe3
+    gpointer key, value;
33afe3
+
33afe3
+    g_hash_table_iter_init(&iter, table);
33afe3
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
33afe3
+        if (!g_list_find_custom(retval, value, (GCompareFunc) strcmp)) {
33afe3
+            retval = g_list_prepend(retval, (char *) value);
33afe3
+        }
33afe3
+    }
33afe3
+
33afe3
+    retval = g_list_sort(retval, (GCompareFunc) strcmp);
33afe3
+    return retval;
33afe3
+}
33afe3
+
33afe3
+static GList *
33afe3
+nodes_with_status(GHashTable *table, const char *status)
33afe3
+{
33afe3
+    GList *retval = NULL;
33afe3
+    GHashTableIter iter;
33afe3
+    gpointer key, value;
33afe3
+
33afe3
+    g_hash_table_iter_init(&iter, table);
33afe3
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
33afe3
+        if (!strcmp((char *) value, status)) {
33afe3
+            retval = g_list_prepend(retval, key);
33afe3
+        }
33afe3
+    }
33afe3
+
33afe3
+    retval = g_list_sort(retval, (GCompareFunc) pcmk__numeric_strcasecmp);
33afe3
+    return retval;
33afe3
+}
33afe3
+
33afe3
+static char *
33afe3
+node_list_to_str(GList *list)
33afe3
+{
33afe3
+    char *retval = NULL;
33afe3
+    size_t len = 0;
33afe3
+
33afe3
+    for (GList *iter = list; iter != NULL; iter = iter->next) {
33afe3
+        pcmk__add_word(&retval, &len, (char *) iter->data);
33afe3
+    }
33afe3
+
33afe3
+    return retval;
33afe3
+}
33afe3
+
33afe3
 static void
33afe3
 clone_header(pcmk__output_t *out, int *rc, pe_resource_t *rsc, clone_variant_data_t *clone_data)
33afe3
 {
33afe3
@@ -710,10 +759,10 @@ pe__clone_default(pcmk__output_t *out, va_list args)
33afe3
     GList *only_node = va_arg(args, GList *);
33afe3
     GList *only_rsc = va_arg(args, GList *);
33afe3
 
33afe3
+    GHashTable *stopped = pcmk__strkey_table(free, free);
33afe3
+
33afe3
     char *list_text = NULL;
33afe3
-    char *stopped_list = NULL;
33afe3
     size_t list_text_len = 0;
33afe3
-    size_t stopped_list_len = 0;
33afe3
 
33afe3
     GList *promoted_list = NULL;
33afe3
     GList *started_list = NULL;
33afe3
@@ -768,7 +817,7 @@ pe__clone_default(pcmk__output_t *out, va_list args)
33afe3
             // List stopped instances when requested (except orphans)
33afe3
             if (!pcmk_is_set(child_rsc->flags, pe_rsc_orphan)
33afe3
                 && pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
33afe3
-                pcmk__add_word(&stopped_list, &stopped_list_len, child_rsc->id);
33afe3
+                g_hash_table_insert(stopped, strdup(child_rsc->id), strdup("Stopped"));
33afe3
             }
33afe3
 
33afe3
         } else if (is_set_recursive(child_rsc, pe_rsc_orphan, TRUE)
33afe3
@@ -822,7 +871,7 @@ pe__clone_default(pcmk__output_t *out, va_list args)
33afe3
     }
33afe3
 
33afe3
     if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
33afe3
-        free(stopped_list);
33afe3
+        g_hash_table_destroy(stopped);
33afe3
         PCMK__OUTPUT_LIST_FOOTER(out, rc);
33afe3
         return pcmk_rc_ok;
33afe3
     }
33afe3
@@ -890,23 +939,15 @@ pe__clone_default(pcmk__output_t *out, va_list args)
33afe3
     }
33afe3
 
33afe3
     if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
33afe3
-        const char *state = "Stopped";
33afe3
-        enum rsc_role_e role = configured_role(rsc);
33afe3
-
33afe3
-        if (role == RSC_ROLE_STOPPED) {
33afe3
-            state = "Stopped (disabled)";
33afe3
-        }
33afe3
-
33afe3
         if (!pcmk_is_set(rsc->flags, pe_rsc_unique)
33afe3
             && (clone_data->clone_max > active_instances)) {
33afe3
 
33afe3
             GList *nIter;
33afe3
             GList *list = g_hash_table_get_values(rsc->allowed_nodes);
33afe3
 
33afe3
-            /* Custom stopped list for non-unique clones */
33afe3
-            free(stopped_list);
33afe3
-            stopped_list = NULL;
33afe3
-            stopped_list_len = 0;
33afe3
+            /* Custom stopped table for non-unique clones */
33afe3
+            g_hash_table_destroy(stopped);
33afe3
+            stopped = pcmk__strkey_table(free, free);
33afe3
 
33afe3
             if (list == NULL) {
33afe3
                 /* Clusters with symmetrical=false haven't calculated allowed_nodes yet
33afe3
@@ -922,19 +963,39 @@ pe__clone_default(pcmk__output_t *out, va_list args)
33afe3
                 if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
33afe3
                     pcmk__str_in_list(node->details->uname, only_node,
33afe3
                                       pcmk__str_star_matches|pcmk__str_casei)) {
33afe3
-                    pcmk__add_word(&stopped_list, &stopped_list_len,
33afe3
-                                   node->details->uname);
33afe3
+                    const char *state = "Stopped";
33afe3
+
33afe3
+                    if (configured_role(rsc) == RSC_ROLE_STOPPED) {
33afe3
+                        state = "Stopped (disabled)";
33afe3
+                    }
33afe3
+
33afe3
+                    g_hash_table_insert(stopped, strdup(node->details->uname),
33afe3
+                                        strdup(state));
33afe3
                 }
33afe3
             }
33afe3
             g_list_free(list);
33afe3
         }
33afe3
 
33afe3
-        if (stopped_list != NULL) {
33afe3
+        if (g_hash_table_size(stopped) > 0) {
33afe3
+            GList *list = sorted_hash_table_values(stopped);
33afe3
+
33afe3
             clone_header(out, &rc, rsc, clone_data);
33afe3
 
33afe3
-            out->list_item(out, NULL, "%s: [ %s ]", state, stopped_list);
33afe3
-            free(stopped_list);
33afe3
-            stopped_list_len = 0;
33afe3
+            for (GList *status_iter = list; status_iter != NULL; status_iter = status_iter->next) {
33afe3
+                const char *status = status_iter->data;
33afe3
+                GList *nodes = nodes_with_status(stopped, status);
33afe3
+                char *str = node_list_to_str(nodes);
33afe3
+
33afe3
+                if (str != NULL) {
33afe3
+                    out->list_item(out, NULL, "%s: [ %s ]", status, str);
33afe3
+                    free(str);
33afe3
+                }
33afe3
+
33afe3
+                g_list_free(nodes);
33afe3
+            }
33afe3
+
33afe3
+            g_list_free(list);
33afe3
+            g_hash_table_destroy(stopped);
33afe3
 
33afe3
         /* If there are no instances of this clone (perhaps because there are no
33afe3
          * nodes configured), simply output the clone header by itself.  This can
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From 0228a64cea412936fb8ee91b0f83f9800048d3ba Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Fri, 19 Nov 2021 10:06:18 -0500
33afe3
Subject: [PATCH 17/21] Feature: scheduler: Display the reason why a clone rsc
33afe3
 probe failed.
33afe3
33afe3
This is similar to the previous commit that adds reasons for primitive
33afe3
resources.
33afe3
33afe3
See: rhbz#1506372
33afe3
---
33afe3
 cts/cli/regression.crm_mon.exp                |  8 +++----
33afe3
 .../summary/failed-probe-clone.summary        | 14 +++++++------
33afe3
 include/crm/pengine/internal.h                |  2 ++
33afe3
 lib/pengine/clone.c                           | 21 +++++++++++++++++--
33afe3
 lib/pengine/utils.c                           |  7 +++++++
33afe3
 5 files changed, 40 insertions(+), 12 deletions(-)
33afe3
33afe3
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
33afe3
index 4333caa11c..5688500ce5 100644
33afe3
--- a/cts/cli/regression.crm_mon.exp
33afe3
+++ b/cts/cli/regression.crm_mon.exp
33afe3
@@ -3479,7 +3479,7 @@ Node List:
33afe3
 Active Resources:
33afe3
   * Clone Set: ping-clone [ping]:
33afe3
     * ping	(ocf:pacemaker:ping):	 Started cluster01
33afe3
-    * ping	(ocf:pacemaker:ping):	 Stopped
33afe3
+    * ping	(ocf:pacemaker:ping):	 Stopped (not installed) 
33afe3
   * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
     * Replica[0]
33afe3
@@ -3663,7 +3663,7 @@ Node List:
33afe3
 Full List of Resources:
33afe3
   * Clone Set: ping-clone [ping]:
33afe3
     * ping	(ocf:pacemaker:ping):	 Started cluster01
33afe3
-    * ping	(ocf:pacemaker:ping):	 Stopped
33afe3
+    * ping	(ocf:pacemaker:ping):	 Stopped (not installed) 
33afe3
   * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
     * Replica[0]
33afe3
@@ -3705,7 +3705,7 @@ Full List of Resources:
33afe3
   * 1/1	(stonith:fence_xvm):	Active cluster01
33afe3
   * Clone Set: ping-clone [ping]:
33afe3
     * ping	(ocf:pacemaker:ping):	 Started cluster01
33afe3
-    * ping	(ocf:pacemaker:ping):	 Stopped
33afe3
+    * ping	(ocf:pacemaker:ping):	 Stopped (not installed) 
33afe3
   * Container bundle set: httpd-bundle [pcmk:http]:
33afe3
     * Replica[0]
33afe3
       * httpd-bundle-ip-192.168.122.131	(ocf:heartbeat:IPaddr2):	 Started cluster02
33afe3
@@ -3886,7 +3886,7 @@ Node List:
33afe3
 Inactive Resources:
33afe3
   * Clone Set: ping-clone [ping]:
33afe3
     * ping	(ocf:pacemaker:ping):	 Started cluster01
33afe3
-    * ping	(ocf:pacemaker:ping):	 Stopped
33afe3
+    * ping	(ocf:pacemaker:ping):	 Stopped (not installed) 
33afe3
   * Resource Group: partially-active-group:
33afe3
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
33afe3
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped (not installed) 
33afe3
diff --git a/cts/scheduler/summary/failed-probe-clone.summary b/cts/scheduler/summary/failed-probe-clone.summary
33afe3
index ca15c302aa..febee14400 100644
33afe3
--- a/cts/scheduler/summary/failed-probe-clone.summary
33afe3
+++ b/cts/scheduler/summary/failed-probe-clone.summary
33afe3
@@ -5,12 +5,13 @@ Current cluster status:
33afe3
   * Full List of Resources:
33afe3
     * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
     * Clone Set: ping-1-clone [ping-1]:
33afe3
-      * Stopped: [ cluster01 cluster02 ]
33afe3
+      * Stopped (not installed): [ cluster01 cluster02 ]
33afe3
     * Clone Set: ping-2-clone [ping-2]:
33afe3
-      * Stopped: [ cluster01 cluster02 ]
33afe3
+      * Stopped: [ cluster02 ]
33afe3
+      * Stopped (not installed): [ cluster01 ]
33afe3
     * Clone Set: ping-3-clone [ping-3]:
33afe3
       * ping-3	(ocf:pacemaker:ping):	 FAILED cluster01
33afe3
-      * Stopped: [ cluster02 ]
33afe3
+      * Stopped (not installed): [ cluster02 ]
33afe3
 
33afe3
 Transition Summary:
33afe3
   * Start      ping-2:0     ( cluster02 )
33afe3
@@ -38,9 +39,10 @@ Revised Cluster Status:
33afe3
   * Full List of Resources:
33afe3
     * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
     * Clone Set: ping-1-clone [ping-1]:
33afe3
-      * Stopped: [ cluster01 cluster02 ]
33afe3
+      * Stopped (not installed): [ cluster01 cluster02 ]
33afe3
     * Clone Set: ping-2-clone [ping-2]:
33afe3
       * Started: [ cluster02 ]
33afe3
-      * Stopped: [ cluster01 ]
33afe3
+      * Stopped (not installed): [ cluster01 ]
33afe3
     * Clone Set: ping-3-clone [ping-3]:
33afe3
-      * Stopped: [ cluster01 cluster02 ]
33afe3
+      * Stopped: [ cluster01 ]
33afe3
+      * Stopped (not installed): [ cluster02 ]
33afe3
diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
33afe3
index 58dd2e8727..2b20da6e5f 100644
33afe3
--- a/include/crm/pengine/internal.h
33afe3
+++ b/include/crm/pengine/internal.h
33afe3
@@ -576,4 +576,6 @@ gboolean pe__native_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean ch
33afe3
 
33afe3
 xmlNode *pe__failed_probe_for_rsc(pe_resource_t *rsc, const char *name);
33afe3
 
33afe3
+const char *pe__clone_child_id(pe_resource_t *rsc);
33afe3
+
33afe3
 #endif
33afe3
diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
33afe3
index 58fb24d24e..ef4bdc0edf 100644
33afe3
--- a/lib/pengine/clone.c
33afe3
+++ b/lib/pengine/clone.c
33afe3
@@ -963,14 +963,23 @@ pe__clone_default(pcmk__output_t *out, va_list args)
33afe3
                 if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
33afe3
                     pcmk__str_in_list(node->details->uname, only_node,
33afe3
                                       pcmk__str_star_matches|pcmk__str_casei)) {
33afe3
+                    xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node->details->uname);
33afe3
                     const char *state = "Stopped";
33afe3
 
33afe3
                     if (configured_role(rsc) == RSC_ROLE_STOPPED) {
33afe3
                         state = "Stopped (disabled)";
33afe3
                     }
33afe3
 
33afe3
-                    g_hash_table_insert(stopped, strdup(node->details->uname),
33afe3
-                                        strdup(state));
33afe3
+                    if (probe_op != NULL) {
33afe3
+                        int rc;
33afe3
+
33afe3
+                        pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
33afe3
+                        g_hash_table_insert(stopped, strdup(node->details->uname),
33afe3
+                                            crm_strdup_printf("Stopped (%s)", services_ocf_exitcode_str(rc)));
33afe3
+                    } else {
33afe3
+                        g_hash_table_insert(stopped, strdup(node->details->uname),
33afe3
+                                            strdup(state));
33afe3
+                    }
33afe3
                 }
33afe3
             }
33afe3
             g_list_free(list);
33afe3
@@ -1113,3 +1122,11 @@ pe__clone_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent
33afe3
 
33afe3
     return !passes;
33afe3
 }
33afe3
+
33afe3
+const char *
33afe3
+pe__clone_child_id(pe_resource_t *rsc)
33afe3
+{
33afe3
+    clone_variant_data_t *clone_data = NULL;
33afe3
+    get_clone_variant_data(clone_data, rsc);
33afe3
+    return ID(clone_data->xml_obj_child);
33afe3
+}
33afe3
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
33afe3
index 3151f0120b..6c4f3b6971 100644
33afe3
--- a/lib/pengine/utils.c
33afe3
+++ b/lib/pengine/utils.c
33afe3
@@ -2573,8 +2573,15 @@ pe__build_rsc_list(pe_working_set_t *data_set, const char *s) {
33afe3
 xmlNode *
33afe3
 pe__failed_probe_for_rsc(pe_resource_t *rsc, const char *name)
33afe3
 {
33afe3
+    pe_resource_t *parent = uber_parent(rsc);
33afe3
     const char *rsc_id = rsc->id;
33afe3
 
33afe3
+    if (rsc->variant == pe_clone) {
33afe3
+        rsc_id = pe__clone_child_id(rsc);
33afe3
+    } else if (parent->variant == pe_clone) {
33afe3
+        rsc_id = pe__clone_child_id(parent);
33afe3
+    }
33afe3
+
33afe3
     for (xmlNode *xml_op = pcmk__xml_first_child(rsc->cluster->failed); xml_op != NULL;
33afe3
          xml_op = pcmk__xml_next(xml_op)) {
33afe3
         const char *value = NULL;
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From cf8b01da93fce87526617fefdcee6eb9f6ecdbd1 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Wed, 24 Nov 2021 10:57:05 -0500
33afe3
Subject: [PATCH 18/21] Test: cts-cli: Update the last-rc-change sed
33afe3
 expression.
33afe3
33afe3
This can now occur in both the XML output (where it's wrapped in double
33afe3
quotes) and the text output (where it's wrapped in single quotes and
33afe3
followed by a comma).  In addition, a plus or minus can occur in the
33afe3
time string.
33afe3
33afe3
The "{0,1}" syntax takes the place of a "?" for marking the optional
33afe3
comma.  In FreeBSD sed, "?" doesn't mean anything special.
33afe3
---
33afe3
 cts/cli/regression.crm_mon.exp | 12 ++++++------
33afe3
 cts/cts-cli.in                 |  2 +-
33afe3
 2 files changed, 7 insertions(+), 7 deletions(-)
33afe3
33afe3
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
33afe3
index 5688500ce5..957758832d 100644
33afe3
--- a/cts/cli/regression.crm_mon.exp
33afe3
+++ b/cts/cli/regression.crm_mon.exp
33afe3
@@ -3497,7 +3497,7 @@ Active Resources:
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
-  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', queued=0ms, exec=33ms
33afe3
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources
33afe3
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
33afe3
@@ -3641,7 +3641,7 @@ Failed Resource Actions:
33afe3
     </node>
33afe3
   </node_history>
33afe3
   <failures>
33afe3
-    <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
+    <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" queued="0" exec="33" interval="0" task="monitor"/>
33afe3
   </failures>
33afe3
   <status code="0" message="OK"/>
33afe3
 </pacemaker-result>
33afe3
@@ -3684,7 +3684,7 @@ Full List of Resources:
33afe3
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped (not installed) 
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
-  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', queued=0ms, exec=33ms
33afe3
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
33afe3
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
33afe3
@@ -3771,7 +3771,7 @@ Operations:
33afe3
       * (1) probe
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
-  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', queued=0ms, exec=33ms
33afe3
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
33afe3
@@ -3850,7 +3850,7 @@ Active Resources:
33afe3
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
-  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', queued=0ms, exec=33ms
33afe3
 =#=#=#= End test: Text output of inactive member of partially active group - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Text output of inactive member of partially active group
33afe3
 =#=#=#= Begin test: Complete brief text output grouped by node, with inactive resources =#=#=#=
33afe3
@@ -3942,7 +3942,7 @@ Operations:
33afe3
       * (1) probe
33afe3
 
33afe3
 Failed Resource Actions:
33afe3
-  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
33afe3
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', queued=0ms, exec=33ms
33afe3
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
33afe3
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
33afe3
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
33afe3
diff --git a/cts/cts-cli.in b/cts/cts-cli.in
33afe3
index 457816afab..72e9a1e912 100755
33afe3
--- a/cts/cts-cli.in
33afe3
+++ b/cts/cts-cli.in
33afe3
@@ -1870,7 +1870,7 @@ for t in $tests; do
33afe3
         -e 's/.*\(unpack_.*\)@.*\.c:[0-9][0-9]*)/\1/g' \
33afe3
         -e 's/.*\(update_validation\)@.*\.c:[0-9][0-9]*)/\1/g' \
33afe3
         -e 's/.*\(apply_upgrade\)@.*\.c:[0-9][0-9]*)/\1/g' \
33afe3
-        -e 's/ last-rc-change=\"[A-Za-z0-9: ]*\"//'\
33afe3
+        -e "s/ last-rc-change=['\"][-+A-Za-z0-9: ]*['\"],\{0,1\}//" \
33afe3
         -e 's|^/tmp/cts-cli\.validity\.bad.xml\.[^:]*:|validity.bad.xml:|'\
33afe3
         -e 's/^Entity: line [0-9][0-9]*: //'\
33afe3
         -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From dea61f1b6507fbc978e040c1555384d8d7ffa9f3 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Wed, 1 Dec 2021 16:23:14 -0500
33afe3
Subject: [PATCH 19/21] Fix: include: Bump feature set to 3.12.0.
33afe3
33afe3
This is for the scheduler handling changing regarding maskable probe
33afe3
failures.
33afe3
33afe3
See: rhbz#1506372.
33afe3
---
33afe3
 include/crm/crm.h | 2 +-
33afe3
 1 file changed, 1 insertion(+), 1 deletion(-)
33afe3
33afe3
diff --git a/include/crm/crm.h b/include/crm/crm.h
33afe3
index 04d2324d75..16b35e9c55 100644
33afe3
--- a/include/crm/crm.h
33afe3
+++ b/include/crm/crm.h
33afe3
@@ -66,7 +66,7 @@ extern "C" {
33afe3
  * >=3.0.13: Fail counts include operation name and interval
33afe3
  * >=3.2.0:  DC supports PCMK_EXEC_INVALID and PCMK_EXEC_NOT_CONNECTED
33afe3
  */
33afe3
-#  define CRM_FEATURE_SET		"3.11.0"
33afe3
+#  define CRM_FEATURE_SET		"3.12.0"
33afe3
 
33afe3
 /* Pacemaker's CPG protocols use fixed-width binary fields for the sender and
33afe3
  * recipient of a CPG message. This imposes an arbitrary limit on cluster node
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From fef2c61ef462c221809dc91467ea1e96d5478c74 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Mon, 6 Dec 2021 16:42:15 -0500
33afe3
Subject: [PATCH 20/21] Feature: scheduler: Handle masked probes in the
33afe3
 scheduler.
33afe3
33afe3
These probe operations get their rc/status codes mapped to not
33afe3
running/done, but still ensures they end up in the list of failed
33afe3
operations so tool output continues to display them properly.
33afe3
33afe3
Note that failures on bundled resources do not get masked.
33afe3
33afe3
There are no test case changes for this patch.
33afe3
33afe3
See: rhbz#1506372.
33afe3
---
33afe3
 lib/pengine/unpack.c | 42 +++++++++++++++++++++++++++++++++++++-----
33afe3
 1 file changed, 37 insertions(+), 5 deletions(-)
33afe3
33afe3
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
33afe3
index b659f319fb..f3583e97d8 100644
33afe3
--- a/lib/pengine/unpack.c
33afe3
+++ b/lib/pengine/unpack.c
33afe3
@@ -3169,6 +3169,11 @@ remap_operation(xmlNode *xml_op, pe_resource_t *rsc, pe_node_t *node,
33afe3
         }
33afe3
     }
33afe3
 
33afe3
+    if (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
33afe3
+        *status = PCMK_EXEC_DONE;
33afe3
+        *rc = PCMK_OCF_NOT_RUNNING;
33afe3
+    }
33afe3
+
33afe3
     /* If the executor reported an operation status of anything but done or
33afe3
      * error, consider that final. But for done or error, we know better whether
33afe3
      * it should be treated as a failure or not, because we know the expected
33afe3
@@ -3567,12 +3572,12 @@ update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, c
33afe3
     CRM_ASSERT(rsc);
33afe3
     CRM_ASSERT(xml_op);
33afe3
 
33afe3
-    if (rc == PCMK_OCF_NOT_RUNNING) {
33afe3
-        clear_past_failure = TRUE;
33afe3
-
33afe3
-    } else if (rc == PCMK_OCF_NOT_INSTALLED) {
33afe3
+    if (rc == PCMK_OCF_NOT_INSTALLED || (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op))) {
33afe3
         rsc->role = RSC_ROLE_STOPPED;
33afe3
 
33afe3
+    } else if (rc == PCMK_OCF_NOT_RUNNING) {
33afe3
+        clear_past_failure = TRUE;
33afe3
+
33afe3
     } else if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)) {
33afe3
         if (last_failure) {
33afe3
             const char *op_key = get_op_key(xml_op);
33afe3
@@ -3661,8 +3666,10 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
33afe3
               pe_working_set_t *data_set)
33afe3
 {
33afe3
     int rc = 0;
33afe3
+    int old_rc = 0;
33afe3
     int task_id = 0;
33afe3
     int target_rc = 0;
33afe3
+    int old_target_rc = 0;
33afe3
     int status = PCMK_EXEC_UNKNOWN;
33afe3
     guint interval_ms = 0;
33afe3
     const char *task = NULL;
33afe3
@@ -3671,6 +3678,7 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
33afe3
     bool expired = false;
33afe3
     pe_resource_t *parent = rsc;
33afe3
     enum action_fail_response failure_strategy = action_fail_recover;
33afe3
+    bool maskable_probe_failure = false;
33afe3
 
33afe3
     CRM_CHECK(rsc && node && xml_op, return);
33afe3
 
33afe3
@@ -3727,10 +3735,22 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
33afe3
         expired = true;
33afe3
     }
33afe3
 
33afe3
+    old_rc = rc;
33afe3
+    old_target_rc = target_rc;
33afe3
+
33afe3
     remap_operation(xml_op, rsc, node, data_set, on_fail, target_rc,
33afe3
                     &rc, &status);
33afe3
 
33afe3
-    if (expired && (rc != target_rc)) {
33afe3
+    maskable_probe_failure = !pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op);
33afe3
+
33afe3
+    if (expired && maskable_probe_failure && old_rc != old_target_rc) {
33afe3
+        if (rsc->role <= RSC_ROLE_STOPPED) {
33afe3
+            rsc->role = RSC_ROLE_UNKNOWN;
33afe3
+        }
33afe3
+
33afe3
+        goto done;
33afe3
+
33afe3
+    } else if (expired && (rc != target_rc)) {
33afe3
         const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
33afe3
 
33afe3
         if (interval_ms == 0) {
33afe3
@@ -3758,6 +3778,18 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
33afe3
         }
33afe3
     }
33afe3
 
33afe3
+    if (maskable_probe_failure) {
33afe3
+        crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
33afe3
+                   services_ocf_exitcode_str(rc), rsc->id, node->details->uname);
33afe3
+        update_resource_state(rsc, node, xml_op, task, target_rc, *last_failure,
33afe3
+                              on_fail, data_set);
33afe3
+        crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
33afe3
+
33afe3
+        record_failed_op(xml_op, node, rsc, data_set);
33afe3
+        resource_location(parent, node, -INFINITY, "masked-probe-failure", data_set);
33afe3
+        goto done;
33afe3
+    }
33afe3
+
33afe3
     switch (status) {
33afe3
         case PCMK_EXEC_CANCELLED:
33afe3
             // Should never happen
33afe3
-- 
33afe3
2.27.0
33afe3
33afe3
33afe3
From ccff6eb60598f389008b0621447056457da79671 Mon Sep 17 00:00:00 2001
33afe3
From: Chris Lumens <clumens@redhat.com>
33afe3
Date: Tue, 4 Jan 2022 10:14:48 -0500
33afe3
Subject: [PATCH 21/21] Test: scheduler: Add tests for expired, masked probe
33afe3
 failures.
33afe3
33afe3
dummy-1 is a stopped resource with an expired masked probe failure.
33afe3
This probe should be rescheduled.  dummy-2 is a started resource with an
33afe3
expired masked probe failure.  This probe should not be rescheduled.
33afe3
---
33afe3
 cts/cts-scheduler.in                          |  1 +
33afe3
 .../dot/expired-failed-probe-primitive.dot    |  8 ++
33afe3
 .../exp/expired-failed-probe-primitive.exp    | 45 ++++++++++++
33afe3
 .../expired-failed-probe-primitive.scores     |  7 ++
33afe3
 .../expired-failed-probe-primitive.summary    | 26 +++++++
33afe3
 .../xml/expired-failed-probe-primitive.xml    | 73 +++++++++++++++++++
33afe3
 6 files changed, 160 insertions(+)
33afe3
 create mode 100644 cts/scheduler/dot/expired-failed-probe-primitive.dot
33afe3
 create mode 100644 cts/scheduler/exp/expired-failed-probe-primitive.exp
33afe3
 create mode 100644 cts/scheduler/scores/expired-failed-probe-primitive.scores
33afe3
 create mode 100644 cts/scheduler/summary/expired-failed-probe-primitive.summary
33afe3
 create mode 100644 cts/scheduler/xml/expired-failed-probe-primitive.xml
33afe3
33afe3
diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
33afe3
index 3abcbc6c9d..7bc41a0936 100644
33afe3
--- a/cts/cts-scheduler.in
33afe3
+++ b/cts/cts-scheduler.in
33afe3
@@ -115,6 +115,7 @@ TESTS = [
33afe3
         [ "probe-pending-node", "Probe (pending node + unmanaged resource)" ],
33afe3
         [ "failed-probe-primitive", "Maskable vs. unmaskable probe failures on primitive resources" ],
33afe3
         [ "failed-probe-clone", "Maskable vs. unmaskable probe failures on cloned resources" ],
33afe3
+        [ "expired-failed-probe-primitive", "Maskable, expired probe failure on primitive resources" ],
33afe3
         [ "standby", "Standby" ],
33afe3
         [ "comments", "Comments" ],
33afe3
     ],
33afe3
diff --git a/cts/scheduler/dot/expired-failed-probe-primitive.dot b/cts/scheduler/dot/expired-failed-probe-primitive.dot
33afe3
new file mode 100644
33afe3
index 0000000000..610c2b8047
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/dot/expired-failed-probe-primitive.dot
33afe3
@@ -0,0 +1,8 @@
33afe3
+ digraph "g" {
33afe3
+"dummy-1_monitor_0 cluster01" -> "dummy-1_start_0 cluster02" [ style = bold]
33afe3
+"dummy-1_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"]
33afe3
+"dummy-1_monitor_0 cluster02" -> "dummy-1_start_0 cluster02" [ style = bold]
33afe3
+"dummy-1_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"]
33afe3
+"dummy-1_start_0 cluster02" [ style=bold color="green" fontcolor="black"]
33afe3
+"dummy-2_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"]
33afe3
+}
33afe3
diff --git a/cts/scheduler/exp/expired-failed-probe-primitive.exp b/cts/scheduler/exp/expired-failed-probe-primitive.exp
33afe3
new file mode 100644
33afe3
index 0000000000..3c2cbfe411
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/exp/expired-failed-probe-primitive.exp
33afe3
@@ -0,0 +1,45 @@
33afe3
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
33afe3
+  <synapse id="0">
33afe3
+    <action_set>
33afe3
+      <rsc_op id="7" operation="start" operation_key="dummy-1_start_0" on_node="cluster02" on_node_uuid="2">
33afe3
+        <primitive id="dummy-1" class="ocf" provider="pacemaker" type="Dummy"/>
33afe3
+        <attributes CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" />
33afe3
+      </rsc_op>
33afe3
+    </action_set>
33afe3
+    <inputs>
33afe3
+      <trigger>
33afe3
+        <rsc_op id="2" operation="monitor" operation_key="dummy-1_monitor_0" on_node="cluster01" on_node_uuid="1"/>
33afe3
+      </trigger>
33afe3
+      <trigger>
33afe3
+        <rsc_op id="4" operation="monitor" operation_key="dummy-1_monitor_0" on_node="cluster02" on_node_uuid="2"/>
33afe3
+      </trigger>
33afe3
+    </inputs>
33afe3
+  </synapse>
33afe3
+  <synapse id="1">
33afe3
+    <action_set>
33afe3
+      <rsc_op id="4" operation="monitor" operation_key="dummy-1_monitor_0" on_node="cluster02" on_node_uuid="2">
33afe3
+        <primitive id="dummy-1" class="ocf" provider="pacemaker" type="Dummy"/>
33afe3
+        <attributes CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
33afe3
+      </rsc_op>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+  <synapse id="2">
33afe3
+    <action_set>
33afe3
+      <rsc_op id="2" operation="monitor" operation_key="dummy-1_monitor_0" on_node="cluster01" on_node_uuid="1">
33afe3
+        <primitive id="dummy-1" class="ocf" provider="pacemaker" type="Dummy"/>
33afe3
+        <attributes CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
33afe3
+      </rsc_op>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+  <synapse id="3">
33afe3
+    <action_set>
33afe3
+      <rsc_op id="3" operation="monitor" operation_key="dummy-2_monitor_0" on_node="cluster01" on_node_uuid="1">
33afe3
+        <primitive id="dummy-2" class="ocf" provider="pacemaker" type="Dummy"/>
33afe3
+        <attributes CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
33afe3
+      </rsc_op>
33afe3
+    </action_set>
33afe3
+    <inputs/>
33afe3
+  </synapse>
33afe3
+</transition_graph>
33afe3
diff --git a/cts/scheduler/scores/expired-failed-probe-primitive.scores b/cts/scheduler/scores/expired-failed-probe-primitive.scores
33afe3
new file mode 100644
33afe3
index 0000000000..51ae5510e6
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/scores/expired-failed-probe-primitive.scores
33afe3
@@ -0,0 +1,7 @@
33afe3
+
33afe3
+pcmk__native_allocate: Fencing allocation score on cluster01: 0
33afe3
+pcmk__native_allocate: Fencing allocation score on cluster02: 0
33afe3
+pcmk__native_allocate: dummy-1 allocation score on cluster01: 0
33afe3
+pcmk__native_allocate: dummy-1 allocation score on cluster02: 0
33afe3
+pcmk__native_allocate: dummy-2 allocation score on cluster01: 0
33afe3
+pcmk__native_allocate: dummy-2 allocation score on cluster02: 0
33afe3
diff --git a/cts/scheduler/summary/expired-failed-probe-primitive.summary b/cts/scheduler/summary/expired-failed-probe-primitive.summary
33afe3
new file mode 100644
33afe3
index 0000000000..ac0604e84f
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/summary/expired-failed-probe-primitive.summary
33afe3
@@ -0,0 +1,26 @@
33afe3
+Current cluster status:
33afe3
+  * Node List:
33afe3
+    * Online: [ cluster01 cluster02 ]
33afe3
+
33afe3
+  * Full List of Resources:
33afe3
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
+    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped
33afe3
+    * dummy-2	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
+
33afe3
+Transition Summary:
33afe3
+  * Start      dummy-1     ( cluster02 )
33afe3
+
33afe3
+Executing Cluster Transition:
33afe3
+  * Resource action: dummy-1         monitor on cluster02
33afe3
+  * Resource action: dummy-1         monitor on cluster01
33afe3
+  * Resource action: dummy-2         monitor on cluster01
33afe3
+  * Resource action: dummy-1         start on cluster02
33afe3
+
33afe3
+Revised Cluster Status:
33afe3
+  * Node List:
33afe3
+    * Online: [ cluster01 cluster02 ]
33afe3
+
33afe3
+  * Full List of Resources:
33afe3
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
33afe3
+    * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
+    * dummy-2	(ocf:pacemaker:Dummy):	 Started cluster02
33afe3
diff --git a/cts/scheduler/xml/expired-failed-probe-primitive.xml b/cts/scheduler/xml/expired-failed-probe-primitive.xml
33afe3
new file mode 100644
33afe3
index 0000000000..684aa73f92
33afe3
--- /dev/null
33afe3
+++ b/cts/scheduler/xml/expired-failed-probe-primitive.xml
33afe3
@@ -0,0 +1,73 @@
33afe3
+<cib crm_feature_set="3.3.0" validate-with="pacemaker-3.3" epoch="1" num_updates="37" admin_epoch="1" cib-last-written="Tue Jan  4 10:00:00 2021" update-origin="cluster01" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2">
33afe3
+  <configuration>
33afe3
+    <crm_config>
33afe3
+      <cluster_property_set id="cib-bootstrap-options">
33afe3
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
33afe3
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-1.e97f9675f.git.el7-e97f9675f"/>
33afe3
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
33afe3
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="test-cluster"/>
33afe3
+        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
33afe3
+        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
33afe3
+      </cluster_property_set>
33afe3
+    </crm_config>
33afe3
+    <nodes>
33afe3
+      <node id="1" uname="cluster01"/>
33afe3
+      <node id="2" uname="cluster02"/>
33afe3
+    </nodes>
33afe3
+    <resources>
33afe3
+      <primitive class="stonith" id="Fencing" type="fence_xvm">
33afe3
+        <instance_attributes id="Fencing-instance_attributes">
33afe3
+          <nvpair id="Fencing-instance_attributes-ip_family" name="ip_family" value="ipv4"/>
33afe3
+        </instance_attributes>
33afe3
+        <operations>
33afe3
+          <op id="Fencing-monitor-interval-60s" interval="60s" name="monitor"/>
33afe3
+        </operations>
33afe3
+      </primitive>
33afe3
+      <primitive class="ocf" id="dummy-1" provider="pacemaker" type="Dummy">
33afe3
+        <meta_attributes id="dummy-1-meta_attributes">
33afe3
+          <nvpair id="dummy-1-meta_attributes-failure-timeout" name="failure-timeout" value="10"/>
33afe3
+        </meta_attributes>
33afe3
+      </primitive>
33afe3
+      <primitive class="ocf" id="dummy-2" provider="pacemaker" type="Dummy">
33afe3
+        <meta_attributes id="dummy-2-meta_attributes">
33afe3
+          <nvpair id="dummy-2-meta_attributes-failure-timeout" name="failure-timeout" value="10"/>
33afe3
+        </meta_attributes>
33afe3
+      </primitive>
33afe3
+    </resources>
33afe3
+    <constraints/>
33afe3
+  </configuration>
33afe3
+  <status>
33afe3
+    <node_state id="1" uname="cluster01" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
33afe3
+      <lrm id="1">
33afe3
+        <lrm_resources>
33afe3
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
33afe3
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="3" rc-code="0" op-status="0" interval="0" last-rc-change="1588951272" exec-time="36" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
33afe3
+            <lrm_rsc_op id="Fencing_monitor_60000" operation_key="Fencing_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="4" rc-code="0" op-status="0" interval="60000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="a88218bb6c7dc47e6586fc75fc2a8d69"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-1_last_failure_0" operation_key="dummy-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="5" rc-code="5" op-status="0" interval="0" last-rc-change="1590608589" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="6" rc-code="5" op-status="0" interval="0" last-rc-change="1590608589" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+        </lrm_resources>
33afe3
+      </lrm>
33afe3
+    </node_state>
33afe3
+    <node_state id="2" uname="cluster02" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
33afe3
+      <lrm id="2">
33afe3
+        <lrm_resources>
33afe3
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
33afe3
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:7;1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1588951263" exec-time="3" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-1_last_failure_0" operation_key="dummy-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="5" op-status="0" interval="0" last-rc-change="1590608589" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+          <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
33afe3
+            <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="7" rc-code="5" op-status="0" interval="0" last-rc-change="1590608589" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+            <lrm_rsc_op id="dummy-2_last_0" operation_key="dummy-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="8" rc-code="0" op-status="0" interval="0" last-rc-change="1590609000" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
33afe3
+          </lrm_resource>
33afe3
+        </lrm_resources>
33afe3
+      </lrm>
33afe3
+    </node_state>
33afe3
+  </status>
33afe3
+</cib>
33afe3
-- 
33afe3
2.27.0
33afe3