Blame SOURCES/010-probe-failures.patch

533c21
From f2e51898735b5e9990464141fc4aea3dd83f5067 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Thu, 4 Nov 2021 14:36:41 -0400
533c21
Subject: [PATCH 01/21] Refactor: scheduler: Use bool in unpack_rsc_op.
533c21
533c21
Previously, we were using bool but TRUE/FALSE.  Instead, use the actual
533c21
values.
533c21
---
533c21
 lib/pengine/unpack.c | 4 ++--
533c21
 1 file changed, 2 insertions(+), 2 deletions(-)
533c21
533c21
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
533c21
index b1e84110a2..ecc7275e15 100644
533c21
--- a/lib/pengine/unpack.c
533c21
+++ b/lib/pengine/unpack.c
533c21
@@ -3671,7 +3671,7 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
533c21
     const char *task = NULL;
533c21
     const char *task_key = NULL;
533c21
     const char *exit_reason = NULL;
533c21
-    bool expired = FALSE;
533c21
+    bool expired = false;
533c21
     pe_resource_t *parent = rsc;
533c21
     enum action_fail_response failure_strategy = action_fail_recover;
533c21
 
533c21
@@ -3727,7 +3727,7 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
533c21
 
533c21
     if ((status != PCMK_EXEC_NOT_INSTALLED)
533c21
         && check_operation_expiry(rsc, node, rc, xml_op, data_set)) {
533c21
-        expired = TRUE;
533c21
+        expired = true;
533c21
     }
533c21
 
533c21
     if (!strcmp(task, CRMD_ACTION_STATUS)) {
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 4c961b8e670d336a368c7fd1535c247e40c6b48e Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Thu, 4 Nov 2021 15:07:01 -0400
533c21
Subject: [PATCH 02/21] Refactor: scheduler: Add functions for determining if
533c21
 an op is a probe.
533c21
533c21
---
533c21
 include/crm/common/util.h                     |  3 +
533c21
 lib/common/operations.c                       | 21 +++++++
533c21
 lib/common/tests/operations/Makefile.am       |  6 +-
533c21
 .../tests/operations/pcmk_is_probe_test.c     | 37 +++++++++++++
533c21
 .../tests/operations/pcmk_xe_is_probe_test.c  | 55 +++++++++++++++++++
533c21
 lib/pengine/unpack.c                          | 12 ++--
533c21
 lib/pengine/utils.c                           |  5 +-
533c21
 7 files changed, 127 insertions(+), 12 deletions(-)
533c21
 create mode 100644 lib/common/tests/operations/pcmk_is_probe_test.c
533c21
 create mode 100644 lib/common/tests/operations/pcmk_xe_is_probe_test.c
533c21
533c21
diff --git a/include/crm/common/util.h b/include/crm/common/util.h
533c21
index 2728b64492..fbea6e560c 100644
533c21
--- a/include/crm/common/util.h
533c21
+++ b/include/crm/common/util.h
533c21
@@ -72,6 +72,9 @@ xmlNode *crm_create_op_xml(xmlNode *parent, const char *prefix,
533c21
                            const char *timeout);
533c21
 #define CRM_DEFAULT_OP_TIMEOUT_S "20s"
533c21
 
533c21
+bool pcmk_is_probe(const char *task, guint interval);
533c21
+bool pcmk_xe_is_probe(xmlNode *xml_op);
533c21
+
533c21
 int compare_version(const char *version1, const char *version2);
533c21
 
533c21
 /* coverity[+kill] */
533c21
diff --git a/lib/common/operations.c b/lib/common/operations.c
533c21
index 366c189702..978df79082 100644
533c21
--- a/lib/common/operations.c
533c21
+++ b/lib/common/operations.c
533c21
@@ -537,3 +537,24 @@ pcmk__is_fencing_action(const char *action)
533c21
 {
533c21
     return pcmk__str_any_of(action, "off", "reboot", "poweroff", NULL);
533c21
 }
533c21
+
533c21
+bool
533c21
+pcmk_is_probe(const char *task, guint interval)
533c21
+{
533c21
+    if (task == NULL) {
533c21
+        return false;
533c21
+    }
533c21
+
533c21
+    return (interval == 0) && pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_none);
533c21
+}
533c21
+
533c21
+bool
533c21
+pcmk_xe_is_probe(xmlNode *xml_op)
533c21
+{
533c21
+    const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
533c21
+    const char *interval_ms_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
533c21
+    int interval_ms;
533c21
+
533c21
+    pcmk__scan_min_int(interval_ms_s, &interval_ms, 0);
533c21
+    return pcmk_is_probe(task, interval_ms);
533c21
+}
533c21
diff --git a/lib/common/tests/operations/Makefile.am b/lib/common/tests/operations/Makefile.am
533c21
index c8814ff0a8..2e3d0b0679 100644
533c21
--- a/lib/common/tests/operations/Makefile.am
533c21
+++ b/lib/common/tests/operations/Makefile.am
533c21
@@ -1,5 +1,5 @@
533c21
 #
533c21
-# Copyright 2020 the Pacemaker project contributors
533c21
+# Copyright 2020-2021 the Pacemaker project contributors
533c21
 #
533c21
 # The version control history for this file may have further details.
533c21
 #
533c21
@@ -12,6 +12,8 @@ LDADD = $(top_builddir)/lib/common/libcrmcommon.la -lcmocka
533c21
 include $(top_srcdir)/mk/tap.mk
533c21
 
533c21
 # Add "_test" to the end of all test program names to simplify .gitignore.
533c21
-check_PROGRAMS = parse_op_key_test
533c21
+check_PROGRAMS = parse_op_key_test \
533c21
+				 pcmk_is_probe_test \
533c21
+				 pcmk_xe_is_probe_test
533c21
 
533c21
 TESTS = $(check_PROGRAMS)
533c21
diff --git a/lib/common/tests/operations/pcmk_is_probe_test.c b/lib/common/tests/operations/pcmk_is_probe_test.c
533c21
new file mode 100644
533c21
index 0000000000..9b449f1a70
533c21
--- /dev/null
533c21
+++ b/lib/common/tests/operations/pcmk_is_probe_test.c
533c21
@@ -0,0 +1,37 @@
533c21
+/*
533c21
+ * Copyright 2021 the Pacemaker project contributors
533c21
+ *
533c21
+ * The version control history for this file may have further details.
533c21
+ *
533c21
+ * This source code is licensed under the GNU Lesser General Public License
533c21
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
533c21
+ */
533c21
+
533c21
+#include <crm_internal.h>
533c21
+
533c21
+#include <stdarg.h>
533c21
+#include <stddef.h>
533c21
+#include <stdint.h>
533c21
+#include <stdlib.h>
533c21
+#include <setjmp.h>
533c21
+#include <cmocka.h>
533c21
+
533c21
+static void
533c21
+is_probe_test(void **state)
533c21
+{
533c21
+    assert_false(pcmk_is_probe(NULL, 0));
533c21
+    assert_false(pcmk_is_probe("", 0));
533c21
+    assert_false(pcmk_is_probe("blahblah", 0));
533c21
+    assert_false(pcmk_is_probe("monitor", 1));
533c21
+    assert_true(pcmk_is_probe("monitor", 0));
533c21
+}
533c21
+
533c21
+int main(int argc, char **argv)
533c21
+{
533c21
+    const struct CMUnitTest tests[] = {
533c21
+        cmocka_unit_test(is_probe_test),
533c21
+    };
533c21
+
533c21
+    cmocka_set_message_output(CM_OUTPUT_TAP);
533c21
+    return cmocka_run_group_tests(tests, NULL, NULL);
533c21
+}
533c21
diff --git a/lib/common/tests/operations/pcmk_xe_is_probe_test.c b/lib/common/tests/operations/pcmk_xe_is_probe_test.c
533c21
new file mode 100644
533c21
index 0000000000..0283d1c145
533c21
--- /dev/null
533c21
+++ b/lib/common/tests/operations/pcmk_xe_is_probe_test.c
533c21
@@ -0,0 +1,55 @@
533c21
+/*
533c21
+ * Copyright 2021 the Pacemaker project contributors
533c21
+ *
533c21
+ * The version control history for this file may have further details.
533c21
+ *
533c21
+ * This source code is licensed under the GNU Lesser General Public License
533c21
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
533c21
+ */
533c21
+
533c21
+#include <crm_internal.h>
533c21
+
533c21
+#include <stdarg.h>
533c21
+#include <stddef.h>
533c21
+#include <stdint.h>
533c21
+#include <stdlib.h>
533c21
+#include <setjmp.h>
533c21
+#include <cmocka.h>
533c21
+
533c21
+static void
533c21
+op_is_probe_test(void **state)
533c21
+{
533c21
+    xmlNode *node = NULL;
533c21
+
533c21
+    assert_false(pcmk_xe_is_probe(NULL));
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op/>");
533c21
+    assert_false(pcmk_xe_is_probe(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation_key=\"blah\" interval=\"30s\"/>");
533c21
+    assert_false(pcmk_xe_is_probe(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"30s\"/>");
533c21
+    assert_false(pcmk_xe_is_probe(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"start\" interval=\"0\"/>");
533c21
+    assert_false(pcmk_xe_is_probe(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\"/>");
533c21
+    assert_true(pcmk_xe_is_probe(node));
533c21
+    free_xml(node);
533c21
+}
533c21
+
533c21
+int main(int argc, char **argv)
533c21
+{
533c21
+    const struct CMUnitTest tests[] = {
533c21
+        cmocka_unit_test(op_is_probe_test),
533c21
+    };
533c21
+
533c21
+    cmocka_set_message_output(CM_OUTPUT_TAP);
533c21
+    return cmocka_run_group_tests(tests, NULL, NULL);
533c21
+}
533c21
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
533c21
index ecc7275e15..7c0c66e696 100644
533c21
--- a/lib/pengine/unpack.c
533c21
+++ b/lib/pengine/unpack.c
533c21
@@ -83,7 +83,6 @@ is_dangling_guest_node(pe_node_t *node)
533c21
     return FALSE;
533c21
 }
533c21
 
533c21
-
533c21
 /*!
533c21
  * \brief Schedule a fence action for a node
533c21
  *
533c21
@@ -2984,7 +2983,6 @@ static void
533c21
 unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * xml_op, xmlNode ** last_failure,
533c21
                       enum action_fail_response * on_fail, pe_working_set_t * data_set)
533c21
 {
533c21
-    guint interval_ms = 0;
533c21
     bool is_probe = false;
533c21
     pe_action_t *action = NULL;
533c21
 
533c21
@@ -2998,10 +2996,7 @@ unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * x
533c21
 
533c21
     *last_failure = xml_op;
533c21
 
533c21
-    crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
533c21
-    if ((interval_ms == 0) && !strcmp(task, CRMD_ACTION_STATUS)) {
533c21
-        is_probe = true;
533c21
-    }
533c21
+    is_probe = pcmk_xe_is_probe(xml_op);
533c21
 
533c21
     if (exit_reason == NULL) {
533c21
         exit_reason = "";
533c21
@@ -3163,8 +3158,9 @@ determine_op_status(
533c21
     }
533c21
 
533c21
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
533c21
-    if ((interval_ms == 0) && !strcmp(task, CRMD_ACTION_STATUS)) {
533c21
-        is_probe = true;
533c21
+    is_probe = pcmk_xe_is_probe(xml_op);
533c21
+
533c21
+    if (is_probe) {
533c21
         task = "probe";
533c21
     }
533c21
 
533c21
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
533c21
index c5eda3898e..07753e173a 100644
533c21
--- a/lib/pengine/utils.c
533c21
+++ b/lib/pengine/utils.c
533c21
@@ -1066,8 +1066,7 @@ unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * contai
533c21
 {
533c21
     int timeout_ms = 0;
533c21
     const char *value = NULL;
533c21
-    bool is_probe = pcmk__str_eq(action->task, RSC_STATUS, pcmk__str_casei)
533c21
-                    && (interval_ms == 0);
533c21
+    bool is_probe = false;
533c21
 #if ENABLE_VERSIONED_ATTRS
533c21
     pe_rsc_action_details_t *rsc_details = NULL;
533c21
 #endif
533c21
@@ -1094,6 +1093,8 @@ unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * contai
533c21
 
533c21
     CRM_CHECK(action && action->rsc, return);
533c21
 
533c21
+    is_probe = pcmk_is_probe(action->task, interval_ms);
533c21
+
533c21
     // Cluster-wide <op_defaults> <meta_attributes>
533c21
     pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS, &rule_data,
533c21
                                action->meta, NULL, FALSE, data_set);
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 09f32df97ab5064a15ba5a1fb3970d5c64ee7b30 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Fri, 19 Nov 2021 14:47:22 -0500
533c21
Subject: [PATCH 03/21] Refactor: scheduler: Move setting interval_ms in
533c21
 determine_op_status.
533c21
533c21
This can now happen in the only place it's being used.
533c21
---
533c21
 lib/pengine/unpack.c | 9 ++++++---
533c21
 1 file changed, 6 insertions(+), 3 deletions(-)
533c21
533c21
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
533c21
index 7c0c66e696..b9986d2462 100644
533c21
--- a/lib/pengine/unpack.c
533c21
+++ b/lib/pengine/unpack.c
533c21
@@ -3142,7 +3142,6 @@ static int
533c21
 determine_op_status(
533c21
     pe_resource_t *rsc, int rc, int target_rc, pe_node_t * node, xmlNode * xml_op, enum action_fail_response * on_fail, pe_working_set_t * data_set) 
533c21
 {
533c21
-    guint interval_ms = 0;
533c21
     bool is_probe = false;
533c21
     int result = PCMK_EXEC_DONE;
533c21
     const char *key = get_op_key(xml_op);
533c21
@@ -3157,7 +3156,6 @@ determine_op_status(
533c21
         exit_reason = "";
533c21
     }
533c21
 
533c21
-    crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
533c21
     is_probe = pcmk_xe_is_probe(xml_op);
533c21
 
533c21
     if (is_probe) {
533c21
@@ -3230,12 +3228,17 @@ determine_op_status(
533c21
             result = PCMK_EXEC_ERROR_FATAL;
533c21
             break;
533c21
 
533c21
-        case PCMK_OCF_UNIMPLEMENT_FEATURE:
533c21
+        case PCMK_OCF_UNIMPLEMENT_FEATURE: {
533c21
+            guint interval_ms = 0;
533c21
+            crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
533c21
+
533c21
             if (interval_ms > 0) {
533c21
                 result = PCMK_EXEC_NOT_SUPPORTED;
533c21
                 break;
533c21
             }
533c21
             // fall through
533c21
+        }
533c21
+
533c21
         case PCMK_OCF_NOT_INSTALLED:
533c21
         case PCMK_OCF_INVALID_PARAM:
533c21
         case PCMK_OCF_INSUFFICIENT_PRIV:
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 6c8f47453afd6c100fddc45187faff17e15f7bfe Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Fri, 19 Nov 2021 14:57:57 -0500
533c21
Subject: [PATCH 04/21] Refactor: scheduler: Add pcmk_xe_mask_failed_probe.
533c21
533c21
Given an xmlNodePtr for a resource operation, this function will
533c21
determine whether it is a failed probe operation that should not be
533c21
displayed in crm_mon (or other places, I suppose) or not.
533c21
---
533c21
 include/crm/common/util.h                     |   1 +
533c21
 lib/common/operations.c                       |  17 ++
533c21
 lib/common/tests/operations/Makefile.am       |   3 +-
533c21
 .../pcmk_xe_mask_probe_failure_test.c         | 162 ++++++++++++++++++
533c21
 4 files changed, 182 insertions(+), 1 deletion(-)
533c21
 create mode 100644 lib/common/tests/operations/pcmk_xe_mask_probe_failure_test.c
533c21
533c21
diff --git a/include/crm/common/util.h b/include/crm/common/util.h
533c21
index fbea6e560c..784069ba1b 100644
533c21
--- a/include/crm/common/util.h
533c21
+++ b/include/crm/common/util.h
533c21
@@ -74,6 +74,7 @@ xmlNode *crm_create_op_xml(xmlNode *parent, const char *prefix,
533c21
 
533c21
 bool pcmk_is_probe(const char *task, guint interval);
533c21
 bool pcmk_xe_is_probe(xmlNode *xml_op);
533c21
+bool pcmk_xe_mask_probe_failure(xmlNode *xml_op);
533c21
 
533c21
 int compare_version(const char *version1, const char *version2);
533c21
 
533c21
diff --git a/lib/common/operations.c b/lib/common/operations.c
533c21
index 978df79082..54482b8863 100644
533c21
--- a/lib/common/operations.c
533c21
+++ b/lib/common/operations.c
533c21
@@ -558,3 +558,20 @@ pcmk_xe_is_probe(xmlNode *xml_op)
533c21
     pcmk__scan_min_int(interval_ms_s, &interval_ms, 0);
533c21
     return pcmk_is_probe(task, interval_ms);
533c21
 }
533c21
+
533c21
+bool
533c21
+pcmk_xe_mask_probe_failure(xmlNode *xml_op)
533c21
+{
533c21
+    int status = PCMK_EXEC_UNKNOWN;
533c21
+    int rc = PCMK_OCF_OK;
533c21
+
533c21
+    if (!pcmk_xe_is_probe(xml_op)) {
533c21
+        return false;
533c21
+    }
533c21
+
533c21
+    crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, &status);
533c21
+    crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &rc);
533c21
+
533c21
+    return rc == PCMK_OCF_NOT_INSTALLED || rc == PCMK_OCF_INVALID_PARAM ||
533c21
+           status == PCMK_EXEC_NOT_INSTALLED;
533c21
+}
533c21
diff --git a/lib/common/tests/operations/Makefile.am b/lib/common/tests/operations/Makefile.am
533c21
index 2e3d0b0679..457c5f7c7a 100644
533c21
--- a/lib/common/tests/operations/Makefile.am
533c21
+++ b/lib/common/tests/operations/Makefile.am
533c21
@@ -14,6 +14,7 @@ include $(top_srcdir)/mk/tap.mk
533c21
 # Add "_test" to the end of all test program names to simplify .gitignore.
533c21
 check_PROGRAMS = parse_op_key_test \
533c21
 				 pcmk_is_probe_test \
533c21
-				 pcmk_xe_is_probe_test
533c21
+				 pcmk_xe_is_probe_test \
533c21
+				 pcmk_xe_mask_probe_failure_test
533c21
 
533c21
 TESTS = $(check_PROGRAMS)
533c21
diff --git a/lib/common/tests/operations/pcmk_xe_mask_probe_failure_test.c b/lib/common/tests/operations/pcmk_xe_mask_probe_failure_test.c
533c21
new file mode 100644
533c21
index 0000000000..a13f6d98f4
533c21
--- /dev/null
533c21
+++ b/lib/common/tests/operations/pcmk_xe_mask_probe_failure_test.c
533c21
@@ -0,0 +1,162 @@
533c21
+/*
533c21
+ * Copyright 2021 the Pacemaker project contributors
533c21
+ *
533c21
+ * The version control history for this file may have further details.
533c21
+ *
533c21
+ * This source code is licensed under the GNU Lesser General Public License
533c21
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
533c21
+ */
533c21
+
533c21
+#include <crm_internal.h>
533c21
+
533c21
+#include <stdarg.h>
533c21
+#include <stddef.h>
533c21
+#include <stdint.h>
533c21
+#include <stdlib.h>
533c21
+#include <setjmp.h>
533c21
+#include <cmocka.h>
533c21
+
533c21
+static void
533c21
+op_is_not_probe_test(void **state) {
533c21
+    xmlNode *node = NULL;
533c21
+
533c21
+    /* Not worth testing this thoroughly since it's just a duplicate of whether
533c21
+     * pcmk_op_is_probe works or not.
533c21
+     */
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"start\" interval=\"0\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+}
533c21
+
533c21
+static void
533c21
+op_does_not_have_right_values_test(void **state) {
533c21
+    xmlNode *node = NULL;
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+}
533c21
+
533c21
+static void
533c21
+check_values_test(void **state) {
533c21
+    xmlNode *node = NULL;
533c21
+
533c21
+    /* PCMK_EXEC_NOT_SUPPORTED */
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"3\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"3\"/>");
533c21
+    assert_true(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    /* PCMK_EXEC_DONE */
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"0\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"2\" op-status=\"0\"/>");
533c21
+    assert_true(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"0\"/>");
533c21
+    assert_true(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"6\" op-status=\"0\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"7\" op-status=\"0\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    /* PCMK_EXEC_NOT_INSTALLED */
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"7\"/>");
533c21
+    assert_true(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"7\"/>");
533c21
+    assert_true(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    /* PCMK_EXEC_ERROR */
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"4\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"2\" op-status=\"4\"/>");
533c21
+    assert_true(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"4\"/>");
533c21
+    assert_true(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"6\" op-status=\"4\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"7\" op-status=\"4\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    /* PCMK_EXEC_ERROR_HARD */
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"5\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"2\" op-status=\"5\"/>");
533c21
+    assert_true(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"5\"/>");
533c21
+    assert_true(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"6\" op-status=\"5\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"7\" op-status=\"5\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    /* PCMK_EXEC_ERROR_FATAL */
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"0\" op-status=\"6\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"2\" op-status=\"6\"/>");
533c21
+    assert_true(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"5\" op-status=\"6\"/>");
533c21
+    assert_true(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"6\" op-status=\"6\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+
533c21
+    node = string2xml("<lrm_rsc_op operation=\"monitor\" interval=\"0\" rc-code=\"7\" op-status=\"6\"/>");
533c21
+    assert_false(pcmk_xe_mask_probe_failure(node));
533c21
+    free_xml(node);
533c21
+}
533c21
+
533c21
+int main(int argc, char **argv)
533c21
+{
533c21
+    const struct CMUnitTest tests[] = {
533c21
+        cmocka_unit_test(op_is_not_probe_test),
533c21
+        cmocka_unit_test(op_does_not_have_right_values_test),
533c21
+        cmocka_unit_test(check_values_test),
533c21
+    };
533c21
+
533c21
+    cmocka_set_message_output(CM_OUTPUT_TAP);
533c21
+    return cmocka_run_group_tests(tests, NULL, NULL);
533c21
+}
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From c9ce1aaf93cd20bb01e80102dda0ffffb07e6472 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Wed, 1 Dec 2021 14:26:31 -0500
533c21
Subject: [PATCH 05/21] Refactor: scheduler: Combine op status and rc remapping
533c21
 into one function.
533c21
533c21
Well, not quite.  Doing the remapping is complicated enough to where it
533c21
makes sense to have them in separate functions.  However, they can both
533c21
be called from a single new function that takes the place of the
533c21
previous two calls in unpack_rsc_op.
533c21
---
533c21
 lib/pengine/unpack.c | 157 ++++++++++++++++++++-----------------------
533c21
 1 file changed, 72 insertions(+), 85 deletions(-)
533c21
533c21
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
533c21
index b9986d2462..b659f319fb 100644
533c21
--- a/lib/pengine/unpack.c
533c21
+++ b/lib/pengine/unpack.c
533c21
@@ -3121,36 +3121,68 @@ unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * x
533c21
 
533c21
 /*!
533c21
  * \internal
533c21
- * \brief Remap operation status based on action result
533c21
+ * \brief Remap informational monitor results and operation status
533c21
  *
533c21
- * Given an action result, determine an appropriate operation status for the
533c21
- * purposes of responding to the action (the status provided by the executor is
533c21
- * not directly usable since the executor does not know what was expected).
533c21
+ * For the monitor results, certain OCF codes are for providing extended information
533c21
+ * to the user about services that aren't yet failed but not entirely healthy either.
533c21
+ * These must be treated as the "normal" result by Pacemaker.
533c21
+ *
533c21
+ * For operation status, the action result can be used to determine an appropriate
533c21
+ * status for the purposes of responding to the action.  The status provided by the
533c21
+ * executor is not directly usable since the executor does not know what was expected.
533c21
  *
533c21
+ * \param[in]     xml_op     Operation history entry XML from CIB status
533c21
  * \param[in,out] rsc        Resource that operation history entry is for
533c21
- * \param[in]     rc         Actual return code of operation
533c21
- * \param[in]     target_rc  Expected return code of operation
533c21
  * \param[in]     node       Node where operation was executed
533c21
- * \param[in]     xml_op     Operation history entry XML from CIB status
533c21
- * \param[in,out] on_fail    What should be done about the result
533c21
  * \param[in]     data_set   Current cluster working set
533c21
+ * \param[in,out] on_fail    What should be done about the result
533c21
+ * \param[in]     target_rc  Expected return code of operation
533c21
+ * \param[in,out] rc         Actual return code of operation
533c21
+ * \param[in,out] status     Operation execution status
533c21
+ *
533c21
+ * \note If the result is remapped and the node is not shutting down or failed,
533c21
+ *       the operation will be recorded in the data set's list of failed operations
533c21
+ *       to highlight it for the user.
533c21
  *
533c21
- * \return Operation status based on return code and action info
533c21
  * \note This may update the resource's current and next role.
533c21
  */
533c21
-static int
533c21
-determine_op_status(
533c21
-    pe_resource_t *rsc, int rc, int target_rc, pe_node_t * node, xmlNode * xml_op, enum action_fail_response * on_fail, pe_working_set_t * data_set) 
533c21
-{
533c21
+static void
533c21
+remap_operation(xmlNode *xml_op, pe_resource_t *rsc, pe_node_t *node,
533c21
+                pe_working_set_t *data_set, enum action_fail_response *on_fail,
533c21
+                int target_rc, int *rc, int *status) {
533c21
     bool is_probe = false;
533c21
-    int result = PCMK_EXEC_DONE;
533c21
-    const char *key = get_op_key(xml_op);
533c21
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
533c21
+    const char *key = get_op_key(xml_op);
533c21
     const char *exit_reason = crm_element_value(xml_op,
533c21
                                                 XML_LRM_ATTR_EXIT_REASON);
533c21
 
533c21
+    if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_none)) {
533c21
+        int remapped_rc = pcmk__effective_rc(*rc);
533c21
+
533c21
+        if (*rc != remapped_rc) {
533c21
+            crm_trace("Remapping monitor result %d to %d", *rc, remapped_rc);
533c21
+            if (!node->details->shutdown || node->details->online) {
533c21
+                record_failed_op(xml_op, node, rsc, data_set);
533c21
+            }
533c21
+
533c21
+            *rc = remapped_rc;
533c21
+        }
533c21
+    }
533c21
+
533c21
+    /* If the executor reported an operation status of anything but done or
533c21
+     * error, consider that final. But for done or error, we know better whether
533c21
+     * it should be treated as a failure or not, because we know the expected
533c21
+     * result.
533c21
+     */
533c21
+    if (*status != PCMK_EXEC_DONE && *status != PCMK_EXEC_ERROR) {
533c21
+        return;
533c21
+    }
533c21
+
533c21
     CRM_ASSERT(rsc);
533c21
-    CRM_CHECK(task != NULL, return PCMK_EXEC_ERROR);
533c21
+    CRM_CHECK(task != NULL,
533c21
+              *status = PCMK_EXEC_ERROR; return);
533c21
+
533c21
+    *status = PCMK_EXEC_DONE;
533c21
 
533c21
     if (exit_reason == NULL) {
533c21
         exit_reason = "";
533c21
@@ -3171,23 +3203,23 @@ determine_op_status(
533c21
          * those versions or processing of saved CIB files from those versions,
533c21
          * so we do not need to care much about this case.
533c21
          */
533c21
-        result = PCMK_EXEC_ERROR;
533c21
+        *status = PCMK_EXEC_ERROR;
533c21
         crm_warn("Expected result not found for %s on %s (corrupt or obsolete CIB?)",
533c21
                  key, node->details->uname);
533c21
 
533c21
-    } else if (target_rc != rc) {
533c21
-        result = PCMK_EXEC_ERROR;
533c21
+    } else if (target_rc != *rc) {
533c21
+        *status = PCMK_EXEC_ERROR;
533c21
         pe_rsc_debug(rsc, "%s on %s: expected %d (%s), got %d (%s%s%s)",
533c21
                      key, node->details->uname,
533c21
                      target_rc, services_ocf_exitcode_str(target_rc),
533c21
-                     rc, services_ocf_exitcode_str(rc),
533c21
+                     *rc, services_ocf_exitcode_str(*rc),
533c21
                      (*exit_reason? ": " : ""), exit_reason);
533c21
     }
533c21
 
533c21
-    switch (rc) {
533c21
+    switch (*rc) {
533c21
         case PCMK_OCF_OK:
533c21
             if (is_probe && (target_rc == PCMK_OCF_NOT_RUNNING)) {
533c21
-                result = PCMK_EXEC_DONE;
533c21
+                *status = PCMK_EXEC_DONE;
533c21
                 pe_rsc_info(rsc, "Probe found %s active on %s at %s",
533c21
                             rsc->id, node->details->uname,
533c21
                             last_change_str(xml_op));
533c21
@@ -3195,10 +3227,10 @@ determine_op_status(
533c21
             break;
533c21
 
533c21
         case PCMK_OCF_NOT_RUNNING:
533c21
-            if (is_probe || (target_rc == rc)
533c21
+            if (is_probe || (target_rc == *rc)
533c21
                 || !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
533c21
 
533c21
-                result = PCMK_EXEC_DONE;
533c21
+                *status = PCMK_EXEC_DONE;
533c21
                 rsc->role = RSC_ROLE_STOPPED;
533c21
 
533c21
                 /* clear any previous failure actions */
533c21
@@ -3208,8 +3240,8 @@ determine_op_status(
533c21
             break;
533c21
 
533c21
         case PCMK_OCF_RUNNING_PROMOTED:
533c21
-            if (is_probe && (rc != target_rc)) {
533c21
-                result = PCMK_EXEC_DONE;
533c21
+            if (is_probe && (*rc != target_rc)) {
533c21
+                *status = PCMK_EXEC_DONE;
533c21
                 pe_rsc_info(rsc,
533c21
                             "Probe found %s active and promoted on %s at %s",
533c21
                             rsc->id, node->details->uname,
533c21
@@ -3221,11 +3253,11 @@ determine_op_status(
533c21
         case PCMK_OCF_DEGRADED_PROMOTED:
533c21
         case PCMK_OCF_FAILED_PROMOTED:
533c21
             rsc->role = RSC_ROLE_PROMOTED;
533c21
-            result = PCMK_EXEC_ERROR;
533c21
+            *status = PCMK_EXEC_ERROR;
533c21
             break;
533c21
 
533c21
         case PCMK_OCF_NOT_CONFIGURED:
533c21
-            result = PCMK_EXEC_ERROR_FATAL;
533c21
+            *status = PCMK_EXEC_ERROR_FATAL;
533c21
             break;
533c21
 
533c21
         case PCMK_OCF_UNIMPLEMENT_FEATURE: {
533c21
@@ -3233,7 +3265,7 @@ determine_op_status(
533c21
             crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
533c21
 
533c21
             if (interval_ms > 0) {
533c21
-                result = PCMK_EXEC_NOT_SUPPORTED;
533c21
+                *status = PCMK_EXEC_NOT_SUPPORTED;
533c21
                 break;
533c21
             }
533c21
             // fall through
533c21
@@ -3248,26 +3280,27 @@ determine_op_status(
533c21
                 pe_proc_err("No further recovery can be attempted for %s "
533c21
                             "because %s on %s failed (%s%s%s) at %s "
533c21
                             CRM_XS " rc=%d id=%s", rsc->id, task,
533c21
-                            node->details->uname, services_ocf_exitcode_str(rc),
533c21
+                            node->details->uname, services_ocf_exitcode_str(*rc),
533c21
                             (*exit_reason? ": " : ""), exit_reason,
533c21
-                            last_change_str(xml_op), rc, ID(xml_op));
533c21
+                            last_change_str(xml_op), *rc, ID(xml_op));
533c21
                 pe__clear_resource_flags(rsc, pe_rsc_managed);
533c21
                 pe__set_resource_flags(rsc, pe_rsc_block);
533c21
             }
533c21
-            result = PCMK_EXEC_ERROR_HARD;
533c21
+            *status = PCMK_EXEC_ERROR_HARD;
533c21
             break;
533c21
 
533c21
         default:
533c21
-            if (result == PCMK_EXEC_DONE) {
533c21
+            if (*status == PCMK_EXEC_DONE) {
533c21
                 crm_info("Treating unknown exit status %d from %s of %s "
533c21
                          "on %s at %s as failure",
533c21
-                         rc, task, rsc->id, node->details->uname,
533c21
+                         *rc, task, rsc->id, node->details->uname,
533c21
                          last_change_str(xml_op));
533c21
-                result = PCMK_EXEC_ERROR;
533c21
+                *status = PCMK_EXEC_ERROR;
533c21
             }
533c21
             break;
533c21
     }
533c21
-    return result;
533c21
+
533c21
+    pe_rsc_trace(rsc, "Remapped %s status to %d", key, *status);
533c21
 }
533c21
 
533c21
 // return TRUE if start or monitor last failure but parameters changed
533c21
@@ -3622,41 +3655,6 @@ update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, c
533c21
     }
533c21
 }
533c21
 
533c21
-/*!
533c21
- * \internal
533c21
- * \brief Remap informational monitor results to usual values
533c21
- *
533c21
- * Certain OCF result codes are for providing extended information to the
533c21
- * user about services that aren't yet failed but not entirely healthy either.
533c21
- * These must be treated as the "normal" result by Pacemaker.
533c21
- *
533c21
- * \param[in] rc        Actual result of a monitor action
533c21
- * \param[in] xml_op    Operation history XML
533c21
- * \param[in] node      Node that operation happened on
533c21
- * \param[in] rsc       Resource that operation happened to
533c21
- * \param[in] data_set  Cluster working set
533c21
- *
533c21
- * \return Result code that pacemaker should use
533c21
- *
533c21
- * \note If the result is remapped, and the node is not shutting down or failed,
533c21
- *       the operation will be recorded in the data set's list of failed
533c21
- *       operations, to highlight it for the user.
533c21
- */
533c21
-static int
533c21
-remap_monitor_rc(int rc, xmlNode *xml_op, const pe_node_t *node,
533c21
-                 const pe_resource_t *rsc, pe_working_set_t *data_set)
533c21
-{
533c21
-    int remapped_rc = pcmk__effective_rc(rc);
533c21
-
533c21
-    if (rc != remapped_rc) {
533c21
-        crm_trace("Remapping monitor result %d to %d", rc, remapped_rc);
533c21
-        if (!node->details->shutdown || node->details->online) {
533c21
-            record_failed_op(xml_op, node, rsc, data_set);
533c21
-        }
533c21
-    }
533c21
-    return remapped_rc;
533c21
-}
533c21
-
533c21
 static void
533c21
 unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
533c21
               xmlNode **last_failure, enum action_fail_response *on_fail,
533c21
@@ -3712,7 +3710,7 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
533c21
                      node->details->uname, rsc->id);
533c21
     }
533c21
 
533c21
-    /* It should be possible to call remap_monitor_rc() first then call
533c21
+    /* It should be possible to call remap_operation() first then call
533c21
      * check_operation_expiry() only if rc != target_rc, because there should
533c21
      * never be a fail count without at least one unexpected result in the
533c21
      * resource history. That would be more efficient by avoiding having to call
533c21
@@ -3729,9 +3727,8 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
533c21
         expired = true;
533c21
     }
533c21
 
533c21
-    if (!strcmp(task, CRMD_ACTION_STATUS)) {
533c21
-        rc = remap_monitor_rc(rc, xml_op, node, rsc, data_set);
533c21
-    }
533c21
+    remap_operation(xml_op, rsc, node, data_set, on_fail, target_rc,
533c21
+                    &rc, &status);
533c21
 
533c21
     if (expired && (rc != target_rc)) {
533c21
         const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
533c21
@@ -3761,16 +3758,6 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
533c21
         }
533c21
     }
533c21
 
533c21
-    /* If the executor reported an operation status of anything but done or
533c21
-     * error, consider that final. But for done or error, we know better whether
533c21
-     * it should be treated as a failure or not, because we know the expected
533c21
-     * result.
533c21
-     */
533c21
-    if(status == PCMK_EXEC_DONE || status == PCMK_EXEC_ERROR) {
533c21
-        status = determine_op_status(rsc, rc, target_rc, node, xml_op, on_fail, data_set);
533c21
-        pe_rsc_trace(rsc, "Remapped %s status to %d", task_key, status);
533c21
-    }
533c21
-
533c21
     switch (status) {
533c21
         case PCMK_EXEC_CANCELLED:
533c21
             // Should never happen
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 9fdca1999872b3930cf18b7d807ddb259f23e8a5 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Fri, 19 Nov 2021 15:08:16 -0500
533c21
Subject: [PATCH 06/21] Test: cts-cli: Add test output for a native resource
533c21
 with a failed probe op.
533c21
533c21
There are no code changes yet to properly handle displaying these
533c21
operations, so the results here just reflect the current handling.
533c21
---
533c21
 cts/cli/crm_mon-partial.xml    | 16 +++++++++++
533c21
 cts/cli/regression.crm_mon.exp | 50 ++++++++++++++++++++++++++--------
533c21
 2 files changed, 55 insertions(+), 11 deletions(-)
533c21
533c21
diff --git a/cts/cli/crm_mon-partial.xml b/cts/cli/crm_mon-partial.xml
533c21
index e6c6894b6f..b7817e4775 100644
533c21
--- a/cts/cli/crm_mon-partial.xml
533c21
+++ b/cts/cli/crm_mon-partial.xml
533c21
@@ -60,6 +60,16 @@
533c21
           </meta_attributes>
533c21
         </primitive>
533c21
       </group>
533c21
+      <primitive class="ocf" id="smart-mon" provider="pacemaker" type="HealthSMART">
533c21
+        <operations>
533c21
+          <op id="smart-mon-monitor-interval-10s" interval="10s" name="monitor" start-delay="0s" timeout="10s"/>
533c21
+          <op id="smart-mon-start-interval-0s" interval="0s" name="start" timeout="10s"/>
533c21
+          <op id="smart-mon-stop-interval-0s" interval="0s" name="stop" timeout="10s"/>
533c21
+        </operations>
533c21
+        <instance_attributes id="smart-mon-instance_attributes">
533c21
+          <nvpair id="smart-mon-instance_attributes-drives" name="drives" value="/dev/nonexistent"/>
533c21
+        </instance_attributes>
533c21
+      </primitive>
533c21
     </resources>
533c21
     <constraints/>
533c21
   </configuration>
533c21
@@ -94,6 +104,9 @@
533c21
           <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
533c21
             <lrm_rsc_op id="dummy-1_last_0" operation_key="dummy-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.6.0" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1599063458" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
           </lrm_resource>
533c21
+          <lrm_resource id="smart-mon" type="HealthSMART" class="ocf" provider="pacemaker">
533c21
+            <lrm_rsc_op id="smart-mon_last_failure_0" operation_key="smart-mon_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="9" rc-code="5" op-status="0" interval="0" last-rc-change="1636490335" exec-time="33" queue-time="0" op-digest="b368e619fcd06788c996f6a2ef2efb6a"/>
533c21
+          </lrm_resource>
533c21
         </lrm_resources>
533c21
       </lrm>
533c21
       <transient_attributes id="2">
533c21
@@ -135,6 +148,9 @@
533c21
             <lrm_rsc_op id="httpd-bundle-1_monitor_30000" operation_key="httpd-bundle-1_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="30000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="7592cb10fa1499772a031adfd385f558"/>
533c21
           </lrm_resource>
533c21
         </lrm_resources>
533c21
+        <lrm_resource id="smart-mon" type="HealthSMART" class="ocf" provider="pacemaker">
533c21
+          <lrm_rsc_op id="smart-mon_last_failure_0" operation_key="smart-mon_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="9" rc-code="5" op-status="0" interval="0" last-rc-change="1636490335" exec-time="33" queue-time="0" op-digest="b368e619fcd06788c996f6a2ef2efb6a"/>
533c21
+        </lrm_resource>
533c21
       </lrm>
533c21
       <transient_attributes id="1">
533c21
         <instance_attributes id="status-1">
533c21
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
533c21
index 8714f917a9..d12dce3ae8 100644
533c21
--- a/cts/cli/regression.crm_mon.exp
533c21
+++ b/cts/cli/regression.crm_mon.exp
533c21
@@ -3470,7 +3470,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 13 resource instances configured (1 DISABLED)
533c21
+  * 14 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3485,6 +3485,9 @@ Active Resources:
533c21
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
   * Resource Group: partially-active-group (1 member inactive):
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
+
533c21
+Failed Resource Actions:
533c21
+  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources
533c21
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
533c21
@@ -3495,7 +3498,7 @@ Active Resources:
533c21
     <last_update time=""/>
533c21
     <last_change time=""/>
533c21
     <nodes_configured number="4"/>
533c21
-    <resources_configured number="13" disabled="1" blocked="0"/>
533c21
+    <resources_configured number="14" disabled="1" blocked="0"/>
533c21
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
533c21
   </summary>
533c21
   <nodes>
533c21
@@ -3548,6 +3551,7 @@ Active Resources:
533c21
       </resource>
533c21
       <resource id="dummy-2" resource_agent="ocf:pacemaker:Dummy" role="Stopped" target_role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
533c21
     </group>
533c21
+    <resource id="smart-mon" resource_agent="ocf:pacemaker:HealthSMART" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
533c21
   </resources>
533c21
   <node_attributes>
533c21
     <node name="cluster01">
533c21
@@ -3574,6 +3578,9 @@ Active Resources:
533c21
       <resource_history id="dummy-1" orphan="false" migration-threshold="1000000">
533c21
         <operation_history call="2" task="start" rc="0" rc_text="ok" exec-time="0ms" queue-time="0ms"/>
533c21
       </resource_history>
533c21
+      <resource_history id="smart-mon" orphan="false" migration-threshold="1000000">
533c21
+        <operation_history call="9" task="probe" rc="5" rc_text="not installed" exec-time="33ms" queue-time="0ms"/>
533c21
+      </resource_history>
533c21
     </node>
533c21
     <node name="cluster01">
533c21
       <resource_history id="Fencing" orphan="false" migration-threshold="1000000">
533c21
@@ -3603,6 +3610,9 @@ Active Resources:
533c21
       </resource_history>
533c21
     </node>
533c21
   </node_history>
533c21
+  <failures>
533c21
+    <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
533c21
+  </failures>
533c21
   <status code="0" message="OK"/>
533c21
 </pacemaker-result>
533c21
 =#=#=#= End test: XML output of partially active resources - OK (0) =#=#=#=
533c21
@@ -3614,7 +3624,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 13 resource instances configured (1 DISABLED)
533c21
+  * 14 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3631,6 +3641,10 @@ Full List of Resources:
533c21
   * Resource Group: partially-active-group:
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 Stopped (disabled)
533c21
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
+
533c21
+Failed Resource Actions:
533c21
+  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
533c21
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
533c21
@@ -3640,13 +3654,14 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 13 resource instances configured (1 DISABLED)
533c21
+  * 14 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
533c21
 
533c21
 Full List of Resources:
533c21
+  * 0/1	(ocf:pacemaker:HealthSMART):	Active
533c21
   * 1/1	(stonith:fence_xvm):	Active cluster01
533c21
   * Clone Set: ping-clone [ping]:
533c21
     * Started: [ cluster01 ]
533c21
@@ -3676,6 +3691,8 @@ Operations:
533c21
       * (3) monitor: interval="30000ms"
533c21
     * dummy-1: migration-threshold=1000000:
533c21
       * (2) start
533c21
+    * smart-mon: migration-threshold=1000000:
533c21
+      * (9) probe
533c21
   * Node: cluster01:
533c21
     * Fencing: migration-threshold=1000000:
533c21
       * (15) start
533c21
@@ -3695,6 +3712,9 @@ Operations:
533c21
   * Node: httpd-bundle-0@cluster02:
533c21
     * httpd: migration-threshold=1000000:
533c21
       * (1) start
533c21
+
533c21
+Failed Resource Actions:
533c21
+  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
533c21
@@ -3704,7 +3724,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 13 resource instances configured (1 DISABLED)
533c21
+  * 14 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3722,7 +3742,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 13 resource instances configured (1 DISABLED)
533c21
+  * 14 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3741,7 +3761,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 13 resource instances configured (1 DISABLED)
533c21
+  * 14 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3759,7 +3779,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 13 resource instances configured (1 DISABLED)
533c21
+  * 14 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3777,7 +3797,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 13 resource instances configured (1 DISABLED)
533c21
+  * 14 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Node cluster01: online:
533c21
@@ -3806,6 +3826,7 @@ Inactive Resources:
533c21
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
   * Resource Group: partially-active-group:
533c21
     * 1/2	(ocf:pacemaker:Dummy):	Active cluster02
533c21
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
 
533c21
 Node Attributes:
533c21
   * Node: cluster01:
533c21
@@ -3826,6 +3847,8 @@ Operations:
533c21
       * (3) monitor: interval="30000ms"
533c21
     * dummy-1: migration-threshold=1000000:
533c21
       * (2) start
533c21
+    * smart-mon: migration-threshold=1000000:
533c21
+      * (9) probe
533c21
   * Node: cluster01:
533c21
     * Fencing: migration-threshold=1000000:
533c21
       * (15) start
533c21
@@ -3845,6 +3868,9 @@ Operations:
533c21
   * Node: httpd-bundle-0@cluster02:
533c21
     * httpd: migration-threshold=1000000:
533c21
       * (1) start
533c21
+
533c21
+Failed Resource Actions:
533c21
+  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
533c21
@@ -3854,7 +3880,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 13 resource instances configured (1 DISABLED)
533c21
+  * 14 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 ]
533c21
@@ -3865,6 +3891,7 @@ Full List of Resources:
533c21
   * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
 =#=#=#= End test: Text output of partially active resources, with inactive resources, filtered by node - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources, filtered by node
533c21
 =#=#=#= Begin test: Text output of partially active resources, filtered by node =#=#=#=
533c21
@@ -3875,7 +3902,7 @@ Full List of Resources:
533c21
     <last_update time=""/>
533c21
     <last_change time=""/>
533c21
     <nodes_configured number="4"/>
533c21
-    <resources_configured number="13" disabled="1" blocked="0"/>
533c21
+    <resources_configured number="14" disabled="1" blocked="0"/>
533c21
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
533c21
   </summary>
533c21
   <nodes>
533c21
@@ -3905,6 +3932,7 @@ Full List of Resources:
533c21
         </resource>
533c21
       </replica>
533c21
     </bundle>
533c21
+    <resource id="smart-mon" resource_agent="ocf:pacemaker:HealthSMART" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
533c21
   </resources>
533c21
   <node_attributes>
533c21
     <node name="cluster01">
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 1c54d0bbb74d066d55a56eae28d1a579b8854604 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Fri, 19 Nov 2021 15:17:52 -0500
533c21
Subject: [PATCH 07/21] Test: cts-cli: Add test output for a cloned resource
533c21
 with a failed probe op.
533c21
533c21
There are no code changes yet to properly handle displaying these
533c21
operations, so the results here just reflect the current handling.
533c21
---
533c21
 cts/cli/crm_mon-partial.xml    |  3 +++
533c21
 cts/cli/regression.crm_mon.exp | 12 ++++++++++++
533c21
 2 files changed, 15 insertions(+)
533c21
533c21
diff --git a/cts/cli/crm_mon-partial.xml b/cts/cli/crm_mon-partial.xml
533c21
index b7817e4775..1f9dc156aa 100644
533c21
--- a/cts/cli/crm_mon-partial.xml
533c21
+++ b/cts/cli/crm_mon-partial.xml
533c21
@@ -107,6 +107,9 @@
533c21
           <lrm_resource id="smart-mon" type="HealthSMART" class="ocf" provider="pacemaker">
533c21
             <lrm_rsc_op id="smart-mon_last_failure_0" operation_key="smart-mon_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="9" rc-code="5" op-status="0" interval="0" last-rc-change="1636490335" exec-time="33" queue-time="0" op-digest="b368e619fcd06788c996f6a2ef2efb6a"/>
533c21
           </lrm_resource>
533c21
+          <lrm_resource id="ping" class="ocf" provider="pacemaker" type="ping">
533c21
+            <lrm_rsc_op id="ping_last_failure_0" operation_key="ping_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="6" rc-code="5" op-status="0" interval="0" last-rc-change="1637259102" exec-time="0" queue-time="0"/>
533c21
+          </lrm_resource>
533c21
         </lrm_resources>
533c21
       </lrm>
533c21
       <transient_attributes id="2">
533c21
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
533c21
index d12dce3ae8..d093bd8106 100644
533c21
--- a/cts/cli/regression.crm_mon.exp
533c21
+++ b/cts/cli/regression.crm_mon.exp
533c21
@@ -3488,6 +3488,7 @@ Active Resources:
533c21
 
533c21
 Failed Resource Actions:
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
+  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources
533c21
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
533c21
@@ -3581,6 +3582,9 @@ Failed Resource Actions:
533c21
       <resource_history id="smart-mon" orphan="false" migration-threshold="1000000">
533c21
         <operation_history call="9" task="probe" rc="5" rc_text="not installed" exec-time="33ms" queue-time="0ms"/>
533c21
       </resource_history>
533c21
+      <resource_history id="ping" orphan="false" migration-threshold="1000000">
533c21
+        <operation_history call="6" task="probe" rc="5" rc_text="not installed" exec-time="0ms" queue-time="0ms"/>
533c21
+      </resource_history>
533c21
     </node>
533c21
     <node name="cluster01">
533c21
       <resource_history id="Fencing" orphan="false" migration-threshold="1000000">
533c21
@@ -3612,6 +3616,7 @@ Failed Resource Actions:
533c21
   </node_history>
533c21
   <failures>
533c21
     <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
533c21
+    <failure op_key="ping_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="6" status="complete" last-rc-change="2021-11-18 13:11:42 -05:00" queued="0" exec="0" interval="0" task="monitor"/>
533c21
   </failures>
533c21
   <status code="0" message="OK"/>
533c21
 </pacemaker-result>
533c21
@@ -3645,6 +3650,7 @@ Full List of Resources:
533c21
 
533c21
 Failed Resource Actions:
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
+  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
533c21
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
533c21
@@ -3693,6 +3699,8 @@ Operations:
533c21
       * (2) start
533c21
     * smart-mon: migration-threshold=1000000:
533c21
       * (9) probe
533c21
+    * ping: migration-threshold=1000000:
533c21
+      * (6) probe
533c21
   * Node: cluster01:
533c21
     * Fencing: migration-threshold=1000000:
533c21
       * (15) start
533c21
@@ -3715,6 +3723,7 @@ Operations:
533c21
 
533c21
 Failed Resource Actions:
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
+  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
533c21
@@ -3849,6 +3858,8 @@ Operations:
533c21
       * (2) start
533c21
     * smart-mon: migration-threshold=1000000:
533c21
       * (9) probe
533c21
+    * ping: migration-threshold=1000000:
533c21
+      * (6) probe
533c21
   * Node: cluster01:
533c21
     * Fencing: migration-threshold=1000000:
533c21
       * (15) start
533c21
@@ -3871,6 +3882,7 @@ Operations:
533c21
 
533c21
 Failed Resource Actions:
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
+  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 9408f08c07eb531ff84b07bf959f3d681ebf2b78 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Fri, 19 Nov 2021 15:48:16 -0500
533c21
Subject: [PATCH 08/21] Test: cts-cli: Change the resources in
533c21
 partially-active-group.
533c21
533c21
dummy-2 is now not running because it failed to start due to an
533c21
unimplemented feature.  I don't know what could possibly be
533c21
unimplemented about a dummy resource, but it's not important.
533c21
533c21
There is also a new dummy-3 resource that acts exactly the same as
533c21
dummy-2.  This preserves checking that the inactive member output can
533c21
still be displayed.
533c21
533c21
There are no code changes yet to properly handle displaying these
533c21
operations, so the results here just reflect the current handling.
533c21
---
533c21
 cts/cli/crm_mon-partial.xml    |  6 +++-
533c21
 cts/cli/regression.crm_mon.exp | 62 +++++++++++++++++++++++-----------
533c21
 2 files changed, 47 insertions(+), 21 deletions(-)
533c21
533c21
diff --git a/cts/cli/crm_mon-partial.xml b/cts/cli/crm_mon-partial.xml
533c21
index 1f9dc156aa..1ce80ea58a 100644
533c21
--- a/cts/cli/crm_mon-partial.xml
533c21
+++ b/cts/cli/crm_mon-partial.xml
533c21
@@ -54,7 +54,8 @@
533c21
       </bundle>
533c21
       <group id="partially-active-group">
533c21
         <primitive class="ocf" id="dummy-1" provider="pacemaker" type="Dummy"/>
533c21
-        <primitive class="ocf" id="dummy-2" provider="pacemaker" type="Dummy">
533c21
+        <primitive class="ocf" id="dummy-2" provider="pacemaker" type="Dummy"/>
533c21
+        <primitive class="ocf" id="dummy-3" provider="pacemaker" type="Dummy">
533c21
           <meta_attributes id="inactive-dummy-meta_attributes">
533c21
             <nvpair id="inactive-dummy-meta_attributes-target-role" name="target-role" value="Stopped"/>
533c21
           </meta_attributes>
533c21
@@ -104,6 +105,9 @@
533c21
           <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
533c21
             <lrm_rsc_op id="dummy-1_last_0" operation_key="dummy-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.6.0" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1599063458" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
           </lrm_resource>
533c21
+          <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:3;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="3" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
           <lrm_resource id="smart-mon" type="HealthSMART" class="ocf" provider="pacemaker">
533c21
             <lrm_rsc_op id="smart-mon_last_failure_0" operation_key="smart-mon_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="9" rc-code="5" op-status="0" interval="0" last-rc-change="1636490335" exec-time="33" queue-time="0" op-digest="b368e619fcd06788c996f6a2ef2efb6a"/>
533c21
           </lrm_resource>
533c21
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
533c21
index d093bd8106..8cf3a1215e 100644
533c21
--- a/cts/cli/regression.crm_mon.exp
533c21
+++ b/cts/cli/regression.crm_mon.exp
533c21
@@ -3470,7 +3470,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 14 resource instances configured (1 DISABLED)
533c21
+  * 15 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3485,8 +3485,10 @@ Active Resources:
533c21
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
   * Resource Group: partially-active-group (1 member inactive):
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
+    * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
 
533c21
 Failed Resource Actions:
533c21
+  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
533c21
@@ -3499,12 +3501,12 @@ Failed Resource Actions:
533c21
     <last_update time=""/>
533c21
     <last_change time=""/>
533c21
     <nodes_configured number="4"/>
533c21
-    <resources_configured number="14" disabled="1" blocked="0"/>
533c21
+    <resources_configured number="15" disabled="1" blocked="0"/>
533c21
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
533c21
   </summary>
533c21
   <nodes>
533c21
     <node name="cluster01" id="1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="false" resources_running="5" type="member"/>
533c21
-    <node name="cluster02" id="2" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="4" type="member"/>
533c21
+    <node name="cluster02" id="2" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="5" type="member"/>
533c21
     <node name="httpd-bundle-0" id="httpd-bundle-0" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="1" type="remote" id_as_resource="httpd-bundle-docker-0"/>
533c21
     <node name="httpd-bundle-1" id="httpd-bundle-1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="0" type="remote" id_as_resource="httpd-bundle-docker-1"/>
533c21
   </nodes>
533c21
@@ -3546,11 +3548,14 @@ Failed Resource Actions:
533c21
         </resource>
533c21
       </replica>
533c21
     </bundle>
533c21
-    <group id="partially-active-group" number_resources="2" managed="true" disabled="false">
533c21
+    <group id="partially-active-group" number_resources="3" managed="true" disabled="false">
533c21
       <resource id="dummy-1" resource_agent="ocf:pacemaker:Dummy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
533c21
         <node name="cluster02" id="2" cached="true"/>
533c21
       </resource>
533c21
-      <resource id="dummy-2" resource_agent="ocf:pacemaker:Dummy" role="Stopped" target_role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
533c21
+      <resource id="dummy-2" resource_agent="ocf:pacemaker:Dummy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="true" failure_ignored="false" nodes_running_on="1">
533c21
+        <node name="cluster02" id="2" cached="true"/>
533c21
+      </resource>
533c21
+      <resource id="dummy-3" resource_agent="ocf:pacemaker:Dummy" role="Stopped" target_role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
533c21
     </group>
533c21
     <resource id="smart-mon" resource_agent="ocf:pacemaker:HealthSMART" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
533c21
   </resources>
533c21
@@ -3579,6 +3584,9 @@ Failed Resource Actions:
533c21
       <resource_history id="dummy-1" orphan="false" migration-threshold="1000000">
533c21
         <operation_history call="2" task="start" rc="0" rc_text="ok" exec-time="0ms" queue-time="0ms"/>
533c21
       </resource_history>
533c21
+      <resource_history id="dummy-2" orphan="false" migration-threshold="1000000">
533c21
+        <operation_history call="2" task="probe" rc="3" rc_text="unimplemented feature" exec-time="33ms" queue-time="0ms"/>
533c21
+      </resource_history>
533c21
       <resource_history id="smart-mon" orphan="false" migration-threshold="1000000">
533c21
         <operation_history call="9" task="probe" rc="5" rc_text="not installed" exec-time="33ms" queue-time="0ms"/>
533c21
       </resource_history>
533c21
@@ -3615,6 +3623,7 @@ Failed Resource Actions:
533c21
     </node>
533c21
   </node_history>
533c21
   <failures>
533c21
+    <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="33" interval="0" task="monitor"/>
533c21
     <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
533c21
     <failure op_key="ping_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="6" status="complete" last-rc-change="2021-11-18 13:11:42 -05:00" queued="0" exec="0" interval="0" task="monitor"/>
533c21
   </failures>
533c21
@@ -3629,7 +3638,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 14 resource instances configured (1 DISABLED)
533c21
+  * 15 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3645,10 +3654,12 @@ Full List of Resources:
533c21
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
   * Resource Group: partially-active-group:
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
-    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped (disabled)
533c21
+    * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
+    * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
533c21
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
 
533c21
 Failed Resource Actions:
533c21
+  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
533c21
@@ -3660,7 +3671,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 14 resource instances configured (1 DISABLED)
533c21
+  * 15 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3676,7 +3687,7 @@ Full List of Resources:
533c21
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
   * Resource Group: partially-active-group:
533c21
-    * 1/2	(ocf:pacemaker:Dummy):	Active cluster02
533c21
+    * 2/3	(ocf:pacemaker:Dummy):	Active cluster02
533c21
 
533c21
 Node Attributes:
533c21
   * Node: cluster01:
533c21
@@ -3697,6 +3708,8 @@ Operations:
533c21
       * (3) monitor: interval="30000ms"
533c21
     * dummy-1: migration-threshold=1000000:
533c21
       * (2) start
533c21
+    * dummy-2: migration-threshold=1000000:
533c21
+      * (2) probe
533c21
     * smart-mon: migration-threshold=1000000:
533c21
       * (9) probe
533c21
     * ping: migration-threshold=1000000:
533c21
@@ -3722,6 +3735,7 @@ Operations:
533c21
       * (1) start
533c21
 
533c21
 Failed Resource Actions:
533c21
+  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
533c21
@@ -3733,7 +3747,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 14 resource instances configured (1 DISABLED)
533c21
+  * 15 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3742,6 +3756,7 @@ Node List:
533c21
 Active Resources:
533c21
   * Resource Group: partially-active-group (1 member inactive):
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
+    * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
 =#=#=#= End test: Text output of partially active group - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active group
533c21
 =#=#=#= Begin test: Text output of partially active group, with inactive resources =#=#=#=
533c21
@@ -3751,7 +3766,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 14 resource instances configured (1 DISABLED)
533c21
+  * 15 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3760,7 +3775,8 @@ Node List:
533c21
 Full List of Resources:
533c21
   * Resource Group: partially-active-group:
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
-    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped (disabled)
533c21
+    * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
+    * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
533c21
 =#=#=#= End test: Text output of partially active group, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active group, with inactive resources
533c21
 =#=#=#= Begin test: Text output of active member of partially active group =#=#=#=
533c21
@@ -3770,7 +3786,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 14 resource instances configured (1 DISABLED)
533c21
+  * 15 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3788,7 +3804,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 14 resource instances configured (1 DISABLED)
533c21
+  * 15 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3796,7 +3812,10 @@ Node List:
533c21
 
533c21
 Active Resources:
533c21
   * Resource Group: partially-active-group (1 member inactive):
533c21
-    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped (disabled)
533c21
+    * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
+
533c21
+Failed Resource Actions:
533c21
+  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
 =#=#=#= End test: Text output of inactive member of partially active group - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of inactive member of partially active group
533c21
 =#=#=#= Begin test: Complete brief text output grouped by node, with inactive resources =#=#=#=
533c21
@@ -3806,7 +3825,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 14 resource instances configured (1 DISABLED)
533c21
+  * 15 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Node cluster01: online:
533c21
@@ -3820,7 +3839,7 @@ Node List:
533c21
     * Resources:
533c21
       * 1	(ocf:heartbeat:IPaddr2):	Active 
533c21
       * 1	(ocf:heartbeat:docker):	Active 
533c21
-      * 1	(ocf:pacemaker:Dummy):	Active 
533c21
+      * 2	(ocf:pacemaker:Dummy):	Active 
533c21
       * 1	(ocf:pacemaker:remote):	Active 
533c21
   * GuestNode httpd-bundle-0@cluster02: online:
533c21
     * Resources:
533c21
@@ -3834,7 +3853,7 @@ Inactive Resources:
533c21
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
   * Resource Group: partially-active-group:
533c21
-    * 1/2	(ocf:pacemaker:Dummy):	Active cluster02
533c21
+    * 2/3	(ocf:pacemaker:Dummy):	Active cluster02
533c21
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
 
533c21
 Node Attributes:
533c21
@@ -3856,6 +3875,8 @@ Operations:
533c21
       * (3) monitor: interval="30000ms"
533c21
     * dummy-1: migration-threshold=1000000:
533c21
       * (2) start
533c21
+    * dummy-2: migration-threshold=1000000:
533c21
+      * (2) probe
533c21
     * smart-mon: migration-threshold=1000000:
533c21
       * (9) probe
533c21
     * ping: migration-threshold=1000000:
533c21
@@ -3881,6 +3902,7 @@ Operations:
533c21
       * (1) start
533c21
 
533c21
 Failed Resource Actions:
533c21
+  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
533c21
@@ -3892,7 +3914,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 14 resource instances configured (1 DISABLED)
533c21
+  * 15 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 ]
533c21
@@ -3914,7 +3936,7 @@ Full List of Resources:
533c21
     <last_update time=""/>
533c21
     <last_change time=""/>
533c21
     <nodes_configured number="4"/>
533c21
-    <resources_configured number="14" disabled="1" blocked="0"/>
533c21
+    <resources_configured number="15" disabled="1" blocked="0"/>
533c21
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
533c21
   </summary>
533c21
   <nodes>
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 85e76b8bdb4de261a9cb4858eeedd49fba0346a1 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Fri, 19 Nov 2021 15:55:51 -0500
533c21
Subject: [PATCH 09/21] Test: cts-cli: Add a failed probe on a new dummy-4
533c21
 resource.
533c21
533c21
This is to verify that these resources which are part of a group are
533c21
displayed properly.  No code changes will be necessary, since groups are
533c21
just several other resources all in the same pile.
533c21
533c21
There are no code changes yet to properly handle displaying these
533c21
operations, so the results here just reflect the current handling.
533c21
---
533c21
 cts/cli/crm_mon-partial.xml    |  4 +++
533c21
 cts/cli/regression.crm_mon.exp | 51 ++++++++++++++++++++++------------
533c21
 2 files changed, 37 insertions(+), 18 deletions(-)
533c21
533c21
diff --git a/cts/cli/crm_mon-partial.xml b/cts/cli/crm_mon-partial.xml
533c21
index 1ce80ea58a..d4d4a70848 100644
533c21
--- a/cts/cli/crm_mon-partial.xml
533c21
+++ b/cts/cli/crm_mon-partial.xml
533c21
@@ -60,6 +60,7 @@
533c21
             <nvpair id="inactive-dummy-meta_attributes-target-role" name="target-role" value="Stopped"/>
533c21
           </meta_attributes>
533c21
         </primitive>
533c21
+        <primitive class="ocf" id="dummy-4" provider="pacemaker" type="Dummy"/>
533c21
       </group>
533c21
       <primitive class="ocf" id="smart-mon" provider="pacemaker" type="HealthSMART">
533c21
         <operations>
533c21
@@ -108,6 +109,9 @@
533c21
           <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
533c21
             <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:3;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="3" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
           </lrm_resource>
533c21
+          <lrm_resource id="dummy-4" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-4_last_failure_0" operation_key="dummy-4_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="21:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;21:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
           <lrm_resource id="smart-mon" type="HealthSMART" class="ocf" provider="pacemaker">
533c21
             <lrm_rsc_op id="smart-mon_last_failure_0" operation_key="smart-mon_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.11.0" transition-key="3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;3:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="9" rc-code="5" op-status="0" interval="0" last-rc-change="1636490335" exec-time="33" queue-time="0" op-digest="b368e619fcd06788c996f6a2ef2efb6a"/>
533c21
           </lrm_resource>
533c21
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
533c21
index 8cf3a1215e..c524b199e3 100644
533c21
--- a/cts/cli/regression.crm_mon.exp
533c21
+++ b/cts/cli/regression.crm_mon.exp
533c21
@@ -3470,7 +3470,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 15 resource instances configured (1 DISABLED)
533c21
+  * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3483,12 +3483,13 @@ Active Resources:
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
-  * Resource Group: partially-active-group (1 member inactive):
533c21
+  * Resource Group: partially-active-group (2 members inactive):
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
 
533c21
 Failed Resource Actions:
533c21
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
+  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
533c21
@@ -3501,7 +3502,7 @@ Failed Resource Actions:
533c21
     <last_update time=""/>
533c21
     <last_change time=""/>
533c21
     <nodes_configured number="4"/>
533c21
-    <resources_configured number="15" disabled="1" blocked="0"/>
533c21
+    <resources_configured number="16" disabled="1" blocked="0"/>
533c21
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
533c21
   </summary>
533c21
   <nodes>
533c21
@@ -3548,7 +3549,7 @@ Failed Resource Actions:
533c21
         </resource>
533c21
       </replica>
533c21
     </bundle>
533c21
-    <group id="partially-active-group" number_resources="3" managed="true" disabled="false">
533c21
+    <group id="partially-active-group" number_resources="4" managed="true" disabled="false">
533c21
       <resource id="dummy-1" resource_agent="ocf:pacemaker:Dummy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
533c21
         <node name="cluster02" id="2" cached="true"/>
533c21
       </resource>
533c21
@@ -3556,6 +3557,7 @@ Failed Resource Actions:
533c21
         <node name="cluster02" id="2" cached="true"/>
533c21
       </resource>
533c21
       <resource id="dummy-3" resource_agent="ocf:pacemaker:Dummy" role="Stopped" target_role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
533c21
+      <resource id="dummy-4" resource_agent="ocf:pacemaker:Dummy" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
533c21
     </group>
533c21
     <resource id="smart-mon" resource_agent="ocf:pacemaker:HealthSMART" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
533c21
   </resources>
533c21
@@ -3587,6 +3589,9 @@ Failed Resource Actions:
533c21
       <resource_history id="dummy-2" orphan="false" migration-threshold="1000000">
533c21
         <operation_history call="2" task="probe" rc="3" rc_text="unimplemented feature" exec-time="33ms" queue-time="0ms"/>
533c21
       </resource_history>
533c21
+      <resource_history id="dummy-4" orphan="false" migration-threshold="1000000">
533c21
+        <operation_history call="2" task="probe" rc="5" rc_text="not installed" exec-time="0ms" queue-time="0ms"/>
533c21
+      </resource_history>
533c21
       <resource_history id="smart-mon" orphan="false" migration-threshold="1000000">
533c21
         <operation_history call="9" task="probe" rc="5" rc_text="not installed" exec-time="33ms" queue-time="0ms"/>
533c21
       </resource_history>
533c21
@@ -3624,6 +3629,7 @@ Failed Resource Actions:
533c21
   </node_history>
533c21
   <failures>
533c21
     <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="33" interval="0" task="monitor"/>
533c21
+    <failure op_key="dummy-4_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="0" interval="0" task="monitor"/>
533c21
     <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
533c21
     <failure op_key="ping_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="6" status="complete" last-rc-change="2021-11-18 13:11:42 -05:00" queued="0" exec="0" interval="0" task="monitor"/>
533c21
   </failures>
533c21
@@ -3638,7 +3644,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 15 resource instances configured (1 DISABLED)
533c21
+  * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3656,10 +3662,12 @@ Full List of Resources:
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
     * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
533c21
+    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped
533c21
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
 
533c21
 Failed Resource Actions:
533c21
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
+  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
533c21
@@ -3671,7 +3679,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 15 resource instances configured (1 DISABLED)
533c21
+  * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3687,7 +3695,7 @@ Full List of Resources:
533c21
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
   * Resource Group: partially-active-group:
533c21
-    * 2/3	(ocf:pacemaker:Dummy):	Active cluster02
533c21
+    * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
533c21
 
533c21
 Node Attributes:
533c21
   * Node: cluster01:
533c21
@@ -3710,6 +3718,8 @@ Operations:
533c21
       * (2) start
533c21
     * dummy-2: migration-threshold=1000000:
533c21
       * (2) probe
533c21
+    * dummy-4: migration-threshold=1000000:
533c21
+      * (2) probe
533c21
     * smart-mon: migration-threshold=1000000:
533c21
       * (9) probe
533c21
     * ping: migration-threshold=1000000:
533c21
@@ -3736,6 +3746,7 @@ Operations:
533c21
 
533c21
 Failed Resource Actions:
533c21
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
+  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
533c21
@@ -3747,14 +3758,14 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 15 resource instances configured (1 DISABLED)
533c21
+  * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
533c21
 
533c21
 Active Resources:
533c21
-  * Resource Group: partially-active-group (1 member inactive):
533c21
+  * Resource Group: partially-active-group (2 members inactive):
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
 =#=#=#= End test: Text output of partially active group - OK (0) =#=#=#=
533c21
@@ -3766,7 +3777,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 15 resource instances configured (1 DISABLED)
533c21
+  * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
@@ -3777,6 +3788,7 @@ Full List of Resources:
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
     * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
533c21
+    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped
533c21
 =#=#=#= End test: Text output of partially active group, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active group, with inactive resources
533c21
 =#=#=#= Begin test: Text output of active member of partially active group =#=#=#=
533c21
@@ -3786,14 +3798,14 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 15 resource instances configured (1 DISABLED)
533c21
+  * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
533c21
 
533c21
 Active Resources:
533c21
-  * Resource Group: partially-active-group (1 member inactive):
533c21
+  * Resource Group: partially-active-group (2 members inactive):
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
 =#=#=#= End test: Text output of active member of partially active group - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of active member of partially active group
533c21
@@ -3804,14 +3816,14 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 15 resource instances configured (1 DISABLED)
533c21
+  * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 cluster02 ]
533c21
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
533c21
 
533c21
 Active Resources:
533c21
-  * Resource Group: partially-active-group (1 member inactive):
533c21
+  * Resource Group: partially-active-group (2 members inactive):
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
 
533c21
 Failed Resource Actions:
533c21
@@ -3825,7 +3837,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 15 resource instances configured (1 DISABLED)
533c21
+  * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Node cluster01: online:
533c21
@@ -3853,7 +3865,7 @@ Inactive Resources:
533c21
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
   * Resource Group: partially-active-group:
533c21
-    * 2/3	(ocf:pacemaker:Dummy):	Active cluster02
533c21
+    * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
533c21
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
 
533c21
 Node Attributes:
533c21
@@ -3877,6 +3889,8 @@ Operations:
533c21
       * (2) start
533c21
     * dummy-2: migration-threshold=1000000:
533c21
       * (2) probe
533c21
+    * dummy-4: migration-threshold=1000000:
533c21
+      * (2) probe
533c21
     * smart-mon: migration-threshold=1000000:
533c21
       * (9) probe
533c21
     * ping: migration-threshold=1000000:
533c21
@@ -3903,6 +3917,7 @@ Operations:
533c21
 
533c21
 Failed Resource Actions:
533c21
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
+  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
533c21
@@ -3914,7 +3929,7 @@ Cluster Summary:
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
-  * 15 resource instances configured (1 DISABLED)
533c21
+  * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
   * Online: [ cluster01 ]
533c21
@@ -3936,7 +3951,7 @@ Full List of Resources:
533c21
     <last_update time=""/>
533c21
     <last_change time=""/>
533c21
     <nodes_configured number="4"/>
533c21
-    <resources_configured number="15" disabled="1" blocked="0"/>
533c21
+    <resources_configured number="16" disabled="1" blocked="0"/>
533c21
     <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" stop-all-resources="false" stonith-timeout-ms="60000" priority-fencing-delay-ms="0"/>
533c21
   </summary>
533c21
   <nodes>
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 206d733b6ce8e0ffcad243d282e8baa8c3ff72b4 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Tue, 23 Nov 2021 14:33:47 -0500
533c21
Subject: [PATCH 10/21] Test: cts-cli: Add test output for a bundle resource
533c21
 with a failed probe op.
533c21
533c21
This just changes the existing failed bundle resource from not starting
533c21
to failing with a reason.
533c21
533c21
There are no code changes yet to properly handle displaying these
533c21
operations, so the results here just reflect the current handling.
533c21
---
533c21
 cts/cli/crm_mon-partial.xml    |  9 ++++++++
533c21
 cts/cli/regression.crm_mon.exp | 40 +++++++++++++++++++++++++---------
533c21
 2 files changed, 39 insertions(+), 10 deletions(-)
533c21
533c21
diff --git a/cts/cli/crm_mon-partial.xml b/cts/cli/crm_mon-partial.xml
533c21
index d4d4a70848..5981fc653c 100644
533c21
--- a/cts/cli/crm_mon-partial.xml
533c21
+++ b/cts/cli/crm_mon-partial.xml
533c21
@@ -178,5 +178,14 @@
533c21
         </lrm_resources>
533c21
       </lrm>
533c21
     </node_state>
533c21
+    <node_state id="httpd-bundle-1" uname="httpd-bundle-1">
533c21
+      <lrm id="httpd-bundle-1">
533c21
+        <lrm_resources>
533c21
+          <lrm_resource id="httpd" class="ocf" provider="heartbeat" type="apache">
533c21
+            <lrm_rsc_op id="httpd_last_failure_0" operation_key="httpd_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="1:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:2;1:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="2" op-status="0" interval="0" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+        </lrm_resources>
533c21
+      </lrm>
533c21
+    </node_state>
533c21
   </status>
533c21
 </cib>
533c21
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
533c21
index c524b199e3..b690a26fb6 100644
533c21
--- a/cts/cli/regression.crm_mon.exp
533c21
+++ b/cts/cli/regression.crm_mon.exp
533c21
@@ -3482,7 +3482,7 @@ Active Resources:
533c21
   * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
+    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
533c21
   * Resource Group: partially-active-group (2 members inactive):
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
@@ -3492,6 +3492,7 @@ Failed Resource Actions:
533c21
   * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
+  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
533c21
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources
533c21
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
533c21
@@ -3509,7 +3510,7 @@ Failed Resource Actions:
533c21
     <node name="cluster01" id="1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="false" resources_running="5" type="member"/>
533c21
     <node name="cluster02" id="2" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="5" type="member"/>
533c21
     <node name="httpd-bundle-0" id="httpd-bundle-0" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="1" type="remote" id_as_resource="httpd-bundle-docker-0"/>
533c21
-    <node name="httpd-bundle-1" id="httpd-bundle-1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="0" type="remote" id_as_resource="httpd-bundle-docker-1"/>
533c21
+    <node name="httpd-bundle-1" id="httpd-bundle-1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="1" type="remote" id_as_resource="httpd-bundle-docker-1"/>
533c21
   </nodes>
533c21
   <resources>
533c21
     <clone id="ping-clone" multi_state="false" unique="false" managed="true" disabled="false" failed="false" failure_ignored="false">
533c21
@@ -3540,7 +3541,9 @@ Failed Resource Actions:
533c21
         <resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf:heartbeat:IPaddr2" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
533c21
           <node name="cluster01" id="1" cached="true"/>
533c21
         </resource>
533c21
-        <resource id="httpd" resource_agent="ocf:heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
533c21
+        <resource id="httpd" resource_agent="ocf:heartbeat:apache" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="true" failure_ignored="false" nodes_running_on="1">
533c21
+          <node name="httpd-bundle-1" id="httpd-bundle-1" cached="true"/>
533c21
+        </resource>
533c21
         <resource id="httpd-bundle-docker-1" resource_agent="ocf:heartbeat:docker" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
533c21
           <node name="cluster01" id="1" cached="true"/>
533c21
         </resource>
533c21
@@ -3626,12 +3629,18 @@ Failed Resource Actions:
533c21
         <operation_history call="1" task="start" rc="0" rc_text="ok" exec-time="0ms" queue-time="0ms"/>
533c21
       </resource_history>
533c21
     </node>
533c21
+    <node name="httpd-bundle-1">
533c21
+      <resource_history id="httpd" orphan="false" migration-threshold="1000000">
533c21
+        <operation_history call="1" task="probe" rc="2" rc_text="invalid parameter" exec-time="0ms" queue-time="0ms"/>
533c21
+      </resource_history>
533c21
+    </node>
533c21
   </node_history>
533c21
   <failures>
533c21
     <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="33" interval="0" task="monitor"/>
533c21
     <failure op_key="dummy-4_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="0" interval="0" task="monitor"/>
533c21
     <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
533c21
     <failure op_key="ping_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="6" status="complete" last-rc-change="2021-11-18 13:11:42 -05:00" queued="0" exec="0" interval="0" task="monitor"/>
533c21
+    <failure op_key="httpd_monitor_0" node="httpd-bundle-1" exitstatus="invalid parameter" exitreason="" exitcode="2" call="1" status="complete" last-rc-change="2020-05-27 15:43:09 -04:00" queued="0" exec="0" interval="0" task="monitor"/>
533c21
   </failures>
533c21
   <status code="0" message="OK"/>
533c21
 </pacemaker-result>
533c21
@@ -3657,7 +3666,7 @@ Full List of Resources:
533c21
   * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
+    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
533c21
   * Resource Group: partially-active-group:
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
@@ -3670,6 +3679,7 @@ Failed Resource Actions:
533c21
   * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
+  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
533c21
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
533c21
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
533c21
@@ -3693,7 +3703,7 @@ Full List of Resources:
533c21
     * Stopped: [ cluster02 ]
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
     * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
+    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
533c21
   * Resource Group: partially-active-group:
533c21
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
533c21
 
533c21
@@ -3743,12 +3753,16 @@ Operations:
533c21
   * Node: httpd-bundle-0@cluster02:
533c21
     * httpd: migration-threshold=1000000:
533c21
       * (1) start
533c21
+  * Node: httpd-bundle-1@cluster01:
533c21
+    * httpd: migration-threshold=1000000:
533c21
+      * (1) probe
533c21
 
533c21
 Failed Resource Actions:
533c21
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
   * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
+  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
533c21
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
533c21
@@ -3856,14 +3870,14 @@ Node List:
533c21
   * GuestNode httpd-bundle-0@cluster02: online:
533c21
     * Resources:
533c21
       * 1	(ocf:heartbeat:apache):	Active 
533c21
+  * GuestNode httpd-bundle-1@cluster01: online:
533c21
+    * Resources:
533c21
+      * 1	(ocf:heartbeat:apache):	Active 
533c21
 
533c21
 Inactive Resources:
533c21
   * Clone Set: ping-clone [ping]:
533c21
     * Started: [ cluster01 ]
533c21
     * Stopped: [ cluster02 ]
533c21
-  * Container bundle set: httpd-bundle [pcmk:http]:
533c21
-    * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
   * Resource Group: partially-active-group:
533c21
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
533c21
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
@@ -3914,12 +3928,16 @@ Operations:
533c21
   * Node: httpd-bundle-0@cluster02:
533c21
     * httpd: migration-threshold=1000000:
533c21
       * (1) start
533c21
+  * Node: httpd-bundle-1@cluster01:
533c21
+    * httpd: migration-threshold=1000000:
533c21
+      * (1) probe
533c21
 
533c21
 Failed Resource Actions:
533c21
   * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
   * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
   * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
   * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
+  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
533c21
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
533c21
@@ -3939,7 +3957,7 @@ Full List of Resources:
533c21
     * Started: [ cluster01 ]
533c21
   * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 Stopped cluster01
533c21
+    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
533c21
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
 =#=#=#= End test: Text output of partially active resources, with inactive resources, filtered by node - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources, filtered by node
533c21
@@ -3972,7 +3990,9 @@ Full List of Resources:
533c21
         <resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf:heartbeat:IPaddr2" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
533c21
           <node name="cluster01" id="1" cached="true"/>
533c21
         </resource>
533c21
-        <resource id="httpd" resource_agent="ocf:heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
533c21
+        <resource id="httpd" resource_agent="ocf:heartbeat:apache" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="true" failure_ignored="false" nodes_running_on="1">
533c21
+          <node name="httpd-bundle-1" id="httpd-bundle-1" cached="true"/>
533c21
+        </resource>
533c21
         <resource id="httpd-bundle-docker-1" resource_agent="ocf:heartbeat:docker" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
533c21
           <node name="cluster01" id="1" cached="true"/>
533c21
         </resource>
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 6240a28d36c0349e3b1d7f52c36106580c53bb01 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Mon, 22 Nov 2021 10:59:10 -0500
533c21
Subject: [PATCH 11/21] Test: cts: Add --show-detail to a couple of the crm_mon
533c21
 tests.
533c21
533c21
This straightens out a couple differences in output between running
533c21
tests locally (where --enable-compat-2.0 is not given, which would
533c21
automatically add --show-detail) and running tests under mock (where
533c21
that option is given).
533c21
533c21
Note that this only really matters for failed resource actions, which
533c21
were not previously output as part of any crm_mon regression test.  It
533c21
is only the patches in this series that have introduced those, and thus
533c21
this difference.
533c21
---
533c21
 cts/cli/regression.crm_mon.exp | 131 ++++++++++++++++++++-------------
533c21
 cts/cts-cli.in                 |  10 +--
533c21
 2 files changed, 83 insertions(+), 58 deletions(-)
533c21
533c21
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
533c21
index b690a26fb6..d7b9d98e2c 100644
533c21
--- a/cts/cli/regression.crm_mon.exp
533c21
+++ b/cts/cli/regression.crm_mon.exp
533c21
@@ -3466,33 +3466,42 @@ Operations:
533c21
 =#=#=#= Begin test: Text output of partially active resources =#=#=#=
533c21
 Cluster Summary:
533c21
   * Stack: corosync
533c21
-  * Current DC: cluster02 (version) - partition with quorum
533c21
+  * Current DC: cluster02 (2) (version) - partition with quorum
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
   * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
-  * Online: [ cluster01 cluster02 ]
533c21
+  * Online: [ cluster01 (1) cluster02 (2) ]
533c21
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
533c21
 
533c21
 Active Resources:
533c21
   * Clone Set: ping-clone [ping]:
533c21
-    * Started: [ cluster01 ]
533c21
+    * ping	(ocf:pacemaker:ping):	 Started cluster01
533c21
+    * ping	(ocf:pacemaker:ping):	 Stopped
533c21
   * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
-    * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
533c21
+    * Replica[0]
533c21
+      * httpd-bundle-ip-192.168.122.131	(ocf:heartbeat:IPaddr2):	 Started cluster02
533c21
+      * httpd	(ocf:heartbeat:apache):	 Started httpd-bundle-0
533c21
+      * httpd-bundle-docker-0	(ocf:heartbeat:docker):	 Started cluster02
533c21
+      * httpd-bundle-0	(ocf:pacemaker:remote):	 Started cluster02
533c21
+    * Replica[1]
533c21
+      * httpd-bundle-ip-192.168.122.132	(ocf:heartbeat:IPaddr2):	 Started cluster01
533c21
+      * httpd	(ocf:heartbeat:apache):	 FAILED httpd-bundle-1
533c21
+      * httpd-bundle-docker-1	(ocf:heartbeat:docker):	 Started cluster01
533c21
+      * httpd-bundle-1	(ocf:pacemaker:remote):	 Started cluster01
533c21
   * Resource Group: partially-active-group (2 members inactive):
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
 
533c21
 Failed Resource Actions:
533c21
-  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
-  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
-  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
-  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
-  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
533c21
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
+  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
533c21
+  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
533c21
+  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
533c21
+  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
533c21
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources
533c21
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
533c21
@@ -3649,24 +3658,32 @@ Failed Resource Actions:
533c21
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources =#=#=#=
533c21
 Cluster Summary:
533c21
   * Stack: corosync
533c21
-  * Current DC: cluster02 (version) - partition with quorum
533c21
+  * Current DC: cluster02 (2) (version) - partition with quorum
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
   * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
-  * Online: [ cluster01 cluster02 ]
533c21
+  * Online: [ cluster01 (1) cluster02 (2) ]
533c21
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
533c21
 
533c21
 Full List of Resources:
533c21
   * Clone Set: ping-clone [ping]:
533c21
-    * Started: [ cluster01 ]
533c21
-    * Stopped: [ cluster02 ]
533c21
+    * ping	(ocf:pacemaker:ping):	 Started cluster01
533c21
+    * ping	(ocf:pacemaker:ping):	 Stopped
533c21
   * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
-    * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
533c21
+    * Replica[0]
533c21
+      * httpd-bundle-ip-192.168.122.131	(ocf:heartbeat:IPaddr2):	 Started cluster02
533c21
+      * httpd	(ocf:heartbeat:apache):	 Started httpd-bundle-0
533c21
+      * httpd-bundle-docker-0	(ocf:heartbeat:docker):	 Started cluster02
533c21
+      * httpd-bundle-0	(ocf:pacemaker:remote):	 Started cluster02
533c21
+    * Replica[1]
533c21
+      * httpd-bundle-ip-192.168.122.132	(ocf:heartbeat:IPaddr2):	 Started cluster01
533c21
+      * httpd	(ocf:heartbeat:apache):	 FAILED httpd-bundle-1
533c21
+      * httpd-bundle-docker-1	(ocf:heartbeat:docker):	 Started cluster01
533c21
+      * httpd-bundle-1	(ocf:pacemaker:remote):	 Started cluster01
533c21
   * Resource Group: partially-active-group:
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
@@ -3675,46 +3692,54 @@ Full List of Resources:
533c21
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
 
533c21
 Failed Resource Actions:
533c21
-  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
-  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
-  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
-  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
-  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
533c21
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
+  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
533c21
+  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
533c21
+  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
533c21
+  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
533c21
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
533c21
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
533c21
 Cluster Summary:
533c21
   * Stack: corosync
533c21
-  * Current DC: cluster02 (version) - partition with quorum
533c21
+  * Current DC: cluster02 (2) (version) - partition with quorum
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
   * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
-  * Online: [ cluster01 cluster02 ]
533c21
+  * Online: [ cluster01 (1) cluster02 (2) ]
533c21
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
533c21
 
533c21
 Full List of Resources:
533c21
   * 0/1	(ocf:pacemaker:HealthSMART):	Active
533c21
   * 1/1	(stonith:fence_xvm):	Active cluster01
533c21
   * Clone Set: ping-clone [ping]:
533c21
-    * Started: [ cluster01 ]
533c21
-    * Stopped: [ cluster02 ]
533c21
+    * ping	(ocf:pacemaker:ping):	 Started cluster01
533c21
+    * ping	(ocf:pacemaker:ping):	 Stopped
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
-    * httpd-bundle-0 (192.168.122.131)	(ocf:heartbeat:apache):	 Started cluster02
533c21
-    * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
533c21
+    * Replica[0]
533c21
+      * httpd-bundle-ip-192.168.122.131	(ocf:heartbeat:IPaddr2):	 Started cluster02
533c21
+      * httpd	(ocf:heartbeat:apache):	 Started httpd-bundle-0
533c21
+      * httpd-bundle-docker-0	(ocf:heartbeat:docker):	 Started cluster02
533c21
+      * httpd-bundle-0	(ocf:pacemaker:remote):	 Started cluster02
533c21
+    * Replica[1]
533c21
+      * httpd-bundle-ip-192.168.122.132	(ocf:heartbeat:IPaddr2):	 Started cluster01
533c21
+      * httpd	(ocf:heartbeat:apache):	 FAILED httpd-bundle-1
533c21
+      * httpd-bundle-docker-1	(ocf:heartbeat:docker):	 Started cluster01
533c21
+      * httpd-bundle-1	(ocf:pacemaker:remote):	 Started cluster01
533c21
   * Resource Group: partially-active-group:
533c21
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
533c21
 
533c21
 Node Attributes:
533c21
-  * Node: cluster01:
533c21
+  * Node: cluster01 (1):
533c21
     * pingd                           	: 1000      
533c21
-  * Node: cluster02:
533c21
+  * Node: cluster02 (2):
533c21
     * pingd                           	: 1000      
533c21
 
533c21
 Operations:
533c21
-  * Node: cluster02:
533c21
+  * Node: cluster02 (2):
533c21
     * httpd-bundle-ip-192.168.122.131: migration-threshold=1000000:
533c21
       * (2) start
533c21
       * (3) monitor: interval="60000ms"
533c21
@@ -3734,7 +3759,7 @@ Operations:
533c21
       * (9) probe
533c21
     * ping: migration-threshold=1000000:
533c21
       * (6) probe
533c21
-  * Node: cluster01:
533c21
+  * Node: cluster01 (1):
533c21
     * Fencing: migration-threshold=1000000:
533c21
       * (15) start
533c21
       * (20) monitor: interval="60000ms"
533c21
@@ -3758,11 +3783,11 @@ Operations:
533c21
       * (1) probe
533c21
 
533c21
 Failed Resource Actions:
533c21
-  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
-  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
-  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
-  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
-  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
533c21
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
+  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
533c21
+  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
533c21
+  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
533c21
+  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
533c21
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
533c21
@@ -3826,14 +3851,14 @@ Active Resources:
533c21
 =#=#=#= Begin test: Text output of inactive member of partially active group =#=#=#=
533c21
 Cluster Summary:
533c21
   * Stack: corosync
533c21
-  * Current DC: cluster02 (version) - partition with quorum
533c21
+  * Current DC: cluster02 (2) (version) - partition with quorum
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
   * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
-  * Online: [ cluster01 cluster02 ]
533c21
+  * Online: [ cluster01 (1) cluster02 (2) ]
533c21
   * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
533c21
 
533c21
 Active Resources:
533c21
@@ -3841,27 +3866,27 @@ Active Resources:
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
 
533c21
 Failed Resource Actions:
533c21
-  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
 =#=#=#= End test: Text output of inactive member of partially active group - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of inactive member of partially active group
533c21
 =#=#=#= Begin test: Complete brief text output grouped by node, with inactive resources =#=#=#=
533c21
 Cluster Summary:
533c21
   * Stack: corosync
533c21
-  * Current DC: cluster02 (version) - partition with quorum
533c21
+  * Current DC: cluster02 (2) (version) - partition with quorum
533c21
   * Last updated:
533c21
   * Last change:
533c21
   * 4 nodes configured
533c21
   * 16 resource instances configured (1 DISABLED)
533c21
 
533c21
 Node List:
533c21
-  * Node cluster01: online:
533c21
+  * Node cluster01 (1): online:
533c21
     * Resources:
533c21
       * 1	(ocf:heartbeat:IPaddr2):	Active 
533c21
       * 1	(ocf:heartbeat:docker):	Active 
533c21
       * 1	(ocf:pacemaker:ping):	Active 
533c21
       * 1	(ocf:pacemaker:remote):	Active 
533c21
       * 1	(stonith:fence_xvm):	Active 
533c21
-  * Node cluster02: online:
533c21
+  * Node cluster02 (2): online:
533c21
     * Resources:
533c21
       * 1	(ocf:heartbeat:IPaddr2):	Active 
533c21
       * 1	(ocf:heartbeat:docker):	Active 
533c21
@@ -3876,20 +3901,20 @@ Node List:
533c21
 
533c21
 Inactive Resources:
533c21
   * Clone Set: ping-clone [ping]:
533c21
-    * Started: [ cluster01 ]
533c21
-    * Stopped: [ cluster02 ]
533c21
+    * ping	(ocf:pacemaker:ping):	 Started cluster01
533c21
+    * ping	(ocf:pacemaker:ping):	 Stopped
533c21
   * Resource Group: partially-active-group:
533c21
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
533c21
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
 
533c21
 Node Attributes:
533c21
-  * Node: cluster01:
533c21
+  * Node: cluster01 (1):
533c21
     * pingd                           	: 1000      
533c21
-  * Node: cluster02:
533c21
+  * Node: cluster02 (2):
533c21
     * pingd                           	: 1000      
533c21
 
533c21
 Operations:
533c21
-  * Node: cluster02:
533c21
+  * Node: cluster02 (2):
533c21
     * httpd-bundle-ip-192.168.122.131: migration-threshold=1000000:
533c21
       * (2) start
533c21
       * (3) monitor: interval="60000ms"
533c21
@@ -3909,7 +3934,7 @@ Operations:
533c21
       * (9) probe
533c21
     * ping: migration-threshold=1000000:
533c21
       * (6) probe
533c21
-  * Node: cluster01:
533c21
+  * Node: cluster01 (1):
533c21
     * Fencing: migration-threshold=1000000:
533c21
       * (15) start
533c21
       * (20) monitor: interval="60000ms"
533c21
@@ -3933,11 +3958,11 @@ Operations:
533c21
       * (1) probe
533c21
 
533c21
 Failed Resource Actions:
533c21
-  * dummy-2 probe on cluster02 returned 'unimplemented feature' at Wed Sep  2 12:17:38 2020 after 33ms
533c21
-  * dummy-4 probe on cluster02 returned 'not installed' at Wed Sep  2 12:17:38 2020
533c21
-  * smart-mon probe on cluster02 returned 'not installed' at Tue Nov  9 15:38:55 2021 after 33ms
533c21
-  * ping probe on cluster02 returned 'not installed' at Thu Nov 18 13:11:42 2021
533c21
-  * httpd probe on httpd-bundle-1 returned 'invalid parameter' at Wed May 27 15:43:09 2020
533c21
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
+  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
533c21
+  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
533c21
+  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
533c21
+  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
533c21
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
533c21
diff --git a/cts/cts-cli.in b/cts/cts-cli.in
533c21
index d32bfb7ed1..457816afab 100755
533c21
--- a/cts/cts-cli.in
533c21
+++ b/cts/cts-cli.in
533c21
@@ -420,7 +420,7 @@ function test_crm_mon() {
533c21
     export CIB_file="$test_home/cli/crm_mon-partial.xml"
533c21
 
533c21
     desc="Text output of partially active resources"
533c21
-    cmd="crm_mon -1"
533c21
+    cmd="crm_mon -1 --show-detail"
533c21
     test_assert $CRM_EX_OK 0
533c21
 
533c21
     desc="XML output of partially active resources"
533c21
@@ -428,13 +428,13 @@ function test_crm_mon() {
533c21
     test_assert_validate $CRM_EX_OK 0
533c21
 
533c21
     desc="Text output of partially active resources, with inactive resources"
533c21
-    cmd="crm_mon -1 -r"
533c21
+    cmd="crm_mon -1 -r --show-detail"
533c21
     test_assert $CRM_EX_OK 0
533c21
 
533c21
     # XML already includes inactive resources
533c21
 
533c21
     desc="Complete brief text output, with inactive resources"
533c21
-    cmd="crm_mon -1 -r --include=all --brief"
533c21
+    cmd="crm_mon -1 -r --include=all --brief --show-detail"
533c21
     test_assert $CRM_EX_OK 0
533c21
 
533c21
     # XML does not have a brief output option
533c21
@@ -452,11 +452,11 @@ function test_crm_mon() {
533c21
     test_assert $CRM_EX_OK 0
533c21
 
533c21
     desc="Text output of inactive member of partially active group"
533c21
-    cmd="crm_mon -1 --resource=dummy-2"
533c21
+    cmd="crm_mon -1 --resource=dummy-2 --show-detail"
533c21
     test_assert $CRM_EX_OK 0
533c21
 
533c21
     desc="Complete brief text output grouped by node, with inactive resources"
533c21
-    cmd="crm_mon -1 -r --include=all --group-by-node --brief"
533c21
+    cmd="crm_mon -1 -r --include=all --group-by-node --brief --show-detail"
533c21
     test_assert $CRM_EX_OK 0
533c21
 
533c21
     desc="Text output of partially active resources, with inactive resources, filtered by node"
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From da14053e5957d84ed0647688d37733adc2f988a3 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Mon, 29 Nov 2021 15:05:42 -0500
533c21
Subject: [PATCH 12/21] Test: scheduler: Add tests for failed probe operations.
533c21
533c21
This adds identical sets of tests for primitive resources and cloned
533c21
resources.  For the moment, the output reflects the current state of the
533c21
code.  No changes have been made to properly handle these operations
533c21
yet.
533c21
533c21
Each set has three resources, and each is set up with a slightly
533c21
different configuration of probe failures:
533c21
533c21
(1) - Maskable probe failure on each node.
533c21
(2) - Maskable probe failure on one node, successful "not running" probe
533c21
      on the other node.  The resource should be started on the node
533c21
      where "not running" was returned.
533c21
(3) - Maskable probe failure on one node, non-maskable probe failure on
533c21
      the other node.  The resource should not be running anywhere, and
533c21
      should be stopped on the node with the non-maskable failure.
533c21
---
533c21
 cts/cts-scheduler.in                          |   2 +
533c21
 cts/scheduler/dot/failed-probe-clone.dot      |  30 ++++
533c21
 cts/scheduler/dot/failed-probe-primitive.dot  |   4 +
533c21
 cts/scheduler/exp/failed-probe-clone.exp      | 141 ++++++++++++++++++
533c21
 cts/scheduler/exp/failed-probe-primitive.exp  |  20 +++
533c21
 .../scores/failed-probe-clone.scores          |  33 ++++
533c21
 .../scores/failed-probe-primitive.scores      |   9 ++
533c21
 .../summary/failed-probe-clone.summary        |  46 ++++++
533c21
 .../summary/failed-probe-primitive.summary    |  27 ++++
533c21
 cts/scheduler/xml/failed-probe-clone.xml      | 110 ++++++++++++++
533c21
 cts/scheduler/xml/failed-probe-primitive.xml  |  71 +++++++++
533c21
 11 files changed, 493 insertions(+)
533c21
 create mode 100644 cts/scheduler/dot/failed-probe-clone.dot
533c21
 create mode 100644 cts/scheduler/dot/failed-probe-primitive.dot
533c21
 create mode 100644 cts/scheduler/exp/failed-probe-clone.exp
533c21
 create mode 100644 cts/scheduler/exp/failed-probe-primitive.exp
533c21
 create mode 100644 cts/scheduler/scores/failed-probe-clone.scores
533c21
 create mode 100644 cts/scheduler/scores/failed-probe-primitive.scores
533c21
 create mode 100644 cts/scheduler/summary/failed-probe-clone.summary
533c21
 create mode 100644 cts/scheduler/summary/failed-probe-primitive.summary
533c21
 create mode 100644 cts/scheduler/xml/failed-probe-clone.xml
533c21
 create mode 100644 cts/scheduler/xml/failed-probe-primitive.xml
533c21
533c21
diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
533c21
index 17fd6cefdf..3abcbc6c9d 100644
533c21
--- a/cts/cts-scheduler.in
533c21
+++ b/cts/cts-scheduler.in
533c21
@@ -113,6 +113,8 @@ TESTS = [
533c21
         [ "probe-3", "Probe (pending node)" ],
533c21
         [ "probe-4", "Probe (pending node + stopped resource)" ],
533c21
         [ "probe-pending-node", "Probe (pending node + unmanaged resource)" ],
533c21
+        [ "failed-probe-primitive", "Maskable vs. unmaskable probe failures on primitive resources" ],
533c21
+        [ "failed-probe-clone", "Maskable vs. unmaskable probe failures on cloned resources" ],
533c21
         [ "standby", "Standby" ],
533c21
         [ "comments", "Comments" ],
533c21
     ],
533c21
diff --git a/cts/scheduler/dot/failed-probe-clone.dot b/cts/scheduler/dot/failed-probe-clone.dot
533c21
new file mode 100644
533c21
index 0000000000..90536b46ed
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/dot/failed-probe-clone.dot
533c21
@@ -0,0 +1,30 @@
533c21
+ digraph "g" {
533c21
+"ping-1_clear_failcount_0 cluster01" [ style=bold color="green" fontcolor="black"]
533c21
+"ping-1_clear_failcount_0 cluster02" [ style=bold color="green" fontcolor="black"]
533c21
+"ping-2-clone_running_0" [ style=bold color="green" fontcolor="orange"]
533c21
+"ping-2-clone_start_0" -> "ping-2-clone_running_0" [ style = bold]
533c21
+"ping-2-clone_start_0" -> "ping-2_start_0 cluster02" [ style = bold]
533c21
+"ping-2-clone_start_0" [ style=bold color="green" fontcolor="orange"]
533c21
+"ping-2_clear_failcount_0 cluster01" [ style=bold color="green" fontcolor="black"]
533c21
+"ping-2_clear_failcount_0 cluster02" [ style=bold color="green" fontcolor="black"]
533c21
+"ping-2_monitor_10000 cluster02" [ style=bold color="green" fontcolor="black"]
533c21
+"ping-2_start_0 cluster02" -> "ping-2-clone_running_0" [ style = bold]
533c21
+"ping-2_start_0 cluster02" -> "ping-2_monitor_10000 cluster02" [ style = bold]
533c21
+"ping-2_start_0 cluster02" [ style=bold color="green" fontcolor="black"]
533c21
+"ping-3-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
533c21
+"ping-3-clone_start_0" -> "ping-3-clone_running_0" [ style = dashed]
533c21
+"ping-3-clone_start_0" -> "ping-3_start_0 <none>" [ style = dashed]
533c21
+"ping-3-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
533c21
+"ping-3-clone_stop_0" -> "ping-3-clone_stopped_0" [ style = bold]
533c21
+"ping-3-clone_stop_0" -> "ping-3_stop_0 cluster01" [ style = bold]
533c21
+"ping-3-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
533c21
+"ping-3-clone_stopped_0" -> "ping-3-clone_start_0" [ style = dashed]
533c21
+"ping-3-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
533c21
+"ping-3_clear_failcount_0 cluster01" [ style=bold color="green" fontcolor="black"]
533c21
+"ping-3_clear_failcount_0 cluster02" [ style=bold color="green" fontcolor="black"]
533c21
+"ping-3_start_0 <none>" -> "ping-3-clone_running_0" [ style = dashed]
533c21
+"ping-3_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
533c21
+"ping-3_stop_0 cluster01" -> "ping-3-clone_stopped_0" [ style = bold]
533c21
+"ping-3_stop_0 cluster01" -> "ping-3_start_0 <none>" [ style = dashed]
533c21
+"ping-3_stop_0 cluster01" [ style=bold color="green" fontcolor="black"]
533c21
+}
533c21
diff --git a/cts/scheduler/dot/failed-probe-primitive.dot b/cts/scheduler/dot/failed-probe-primitive.dot
533c21
new file mode 100644
533c21
index 0000000000..6e0c83216a
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/dot/failed-probe-primitive.dot
533c21
@@ -0,0 +1,4 @@
533c21
+ digraph "g" {
533c21
+"dummy-2_start_0 cluster02" [ style=bold color="green" fontcolor="black"]
533c21
+"dummy-3_stop_0 cluster01" [ style=bold color="green" fontcolor="black"]
533c21
+}
533c21
diff --git a/cts/scheduler/exp/failed-probe-clone.exp b/cts/scheduler/exp/failed-probe-clone.exp
533c21
new file mode 100644
533c21
index 0000000000..6be18935bf
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/exp/failed-probe-clone.exp
533c21
@@ -0,0 +1,141 @@
533c21
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
533c21
+  <synapse id="0">
533c21
+    <action_set>
533c21
+      <crm_event id="6" operation="clear_failcount" operation_key="ping-1_clear_failcount_0" internal_operation_key="ping-1:0_clear_failcount_0" on_node="cluster02" on_node_uuid="2">
533c21
+        <primitive id="ping-1" long-id="ping-1:0" class="ocf" provider="pacemaker" type="ping"/>
533c21
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.1" multiplier="1000"/>
533c21
+      </crm_event>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+  <synapse id="1">
533c21
+    <action_set>
533c21
+      <crm_event id="2" operation="clear_failcount" operation_key="ping-1_clear_failcount_0" internal_operation_key="ping-1:0_clear_failcount_0" on_node="cluster01" on_node_uuid="1">
533c21
+        <primitive id="ping-1" long-id="ping-1:0" class="ocf" provider="pacemaker" type="ping"/>
533c21
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.1" multiplier="1000"/>
533c21
+      </crm_event>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+  <synapse id="2">
533c21
+    <action_set>
533c21
+      <rsc_op id="17" operation="monitor" operation_key="ping-2_monitor_10000" internal_operation_key="ping-2:0_monitor_10000" on_node="cluster02" on_node_uuid="2">
533c21
+        <primitive id="ping-2" long-id="ping-2:0" class="ocf" provider="pacemaker" type="ping"/>
533c21
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_timeout="60000"  dampen="5s" host_list="192.168.122.2" multiplier="1000"/>
533c21
+      </rsc_op>
533c21
+    </action_set>
533c21
+    <inputs>
533c21
+      <trigger>
533c21
+        <rsc_op id="16" operation="start" operation_key="ping-2_start_0" internal_operation_key="ping-2:0_start_0" on_node="cluster02" on_node_uuid="2"/>
533c21
+      </trigger>
533c21
+    </inputs>
533c21
+  </synapse>
533c21
+  <synapse id="3">
533c21
+    <action_set>
533c21
+      <rsc_op id="16" operation="start" operation_key="ping-2_start_0" internal_operation_key="ping-2:0_start_0" on_node="cluster02" on_node_uuid="2">
533c21
+        <primitive id="ping-2" long-id="ping-2:0" class="ocf" provider="pacemaker" type="ping"/>
533c21
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_timeout="60000"  dampen="5s" host_list="192.168.122.2" multiplier="1000"/>
533c21
+      </rsc_op>
533c21
+    </action_set>
533c21
+    <inputs>
533c21
+      <trigger>
533c21
+        <pseudo_event id="18" operation="start" operation_key="ping-2-clone_start_0"/>
533c21
+      </trigger>
533c21
+    </inputs>
533c21
+  </synapse>
533c21
+  <synapse id="4">
533c21
+    <action_set>
533c21
+      <crm_event id="7" operation="clear_failcount" operation_key="ping-2_clear_failcount_0" internal_operation_key="ping-2:0_clear_failcount_0" on_node="cluster02" on_node_uuid="2">
533c21
+        <primitive id="ping-2" long-id="ping-2:0" class="ocf" provider="pacemaker" type="ping"/>
533c21
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.2" multiplier="1000"/>
533c21
+      </crm_event>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+  <synapse id="5">
533c21
+    <action_set>
533c21
+      <crm_event id="3" operation="clear_failcount" operation_key="ping-2_clear_failcount_0" internal_operation_key="ping-2:0_clear_failcount_0" on_node="cluster01" on_node_uuid="1">
533c21
+        <primitive id="ping-2" long-id="ping-2:0" class="ocf" provider="pacemaker" type="ping"/>
533c21
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.2" multiplier="1000"/>
533c21
+      </crm_event>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+  <synapse id="6" priority="1000000">
533c21
+    <action_set>
533c21
+      <pseudo_event id="19" operation="running" operation_key="ping-2-clone_running_0">
533c21
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
533c21
+      </pseudo_event>
533c21
+    </action_set>
533c21
+    <inputs>
533c21
+      <trigger>
533c21
+        <rsc_op id="16" operation="start" operation_key="ping-2_start_0" internal_operation_key="ping-2:0_start_0" on_node="cluster02" on_node_uuid="2"/>
533c21
+      </trigger>
533c21
+      <trigger>
533c21
+        <pseudo_event id="18" operation="start" operation_key="ping-2-clone_start_0"/>
533c21
+      </trigger>
533c21
+    </inputs>
533c21
+  </synapse>
533c21
+  <synapse id="7">
533c21
+    <action_set>
533c21
+      <pseudo_event id="18" operation="start" operation_key="ping-2-clone_start_0">
533c21
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
533c21
+      </pseudo_event>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+  <synapse id="8">
533c21
+    <action_set>
533c21
+      <rsc_op id="5" operation="stop" operation_key="ping-3_stop_0" internal_operation_key="ping-3:0_stop_0" on_node="cluster01" on_node_uuid="1">
533c21
+        <primitive id="ping-3" long-id="ping-3:0" class="ocf" provider="pacemaker" type="ping"/>
533c21
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.3" multiplier="1000"/>
533c21
+      </rsc_op>
533c21
+    </action_set>
533c21
+    <inputs>
533c21
+      <trigger>
533c21
+        <pseudo_event id="24" operation="stop" operation_key="ping-3-clone_stop_0"/>
533c21
+      </trigger>
533c21
+    </inputs>
533c21
+  </synapse>
533c21
+  <synapse id="9">
533c21
+    <action_set>
533c21
+      <crm_event id="4" operation="clear_failcount" operation_key="ping-3_clear_failcount_0" internal_operation_key="ping-3:0_clear_failcount_0" on_node="cluster01" on_node_uuid="1">
533c21
+        <primitive id="ping-3" long-id="ping-3:0" class="ocf" provider="pacemaker" type="ping"/>
533c21
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.3" multiplier="1000"/>
533c21
+      </crm_event>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+  <synapse id="10">
533c21
+    <action_set>
533c21
+      <crm_event id="8" operation="clear_failcount" operation_key="ping-3_clear_failcount_0" internal_operation_key="ping-3:1_clear_failcount_0" on_node="cluster02" on_node_uuid="2">
533c21
+        <primitive id="ping-3" long-id="ping-3:1" class="ocf" provider="pacemaker" type="ping"/>
533c21
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000"  dampen="5s" host_list="192.168.122.3" multiplier="1000"/>
533c21
+      </crm_event>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+  <synapse id="11" priority="1000000">
533c21
+    <action_set>
533c21
+      <pseudo_event id="25" operation="stopped" operation_key="ping-3-clone_stopped_0">
533c21
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
533c21
+      </pseudo_event>
533c21
+    </action_set>
533c21
+    <inputs>
533c21
+      <trigger>
533c21
+        <rsc_op id="5" operation="stop" operation_key="ping-3_stop_0" internal_operation_key="ping-3:0_stop_0" on_node="cluster01" on_node_uuid="1"/>
533c21
+      </trigger>
533c21
+      <trigger>
533c21
+        <pseudo_event id="24" operation="stop" operation_key="ping-3-clone_stop_0"/>
533c21
+      </trigger>
533c21
+    </inputs>
533c21
+  </synapse>
533c21
+  <synapse id="12">
533c21
+    <action_set>
533c21
+      <pseudo_event id="24" operation="stop" operation_key="ping-3-clone_stop_0">
533c21
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
533c21
+      </pseudo_event>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+</transition_graph>
533c21
diff --git a/cts/scheduler/exp/failed-probe-primitive.exp b/cts/scheduler/exp/failed-probe-primitive.exp
533c21
new file mode 100644
533c21
index 0000000000..d0d8aa44dc
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/exp/failed-probe-primitive.exp
533c21
@@ -0,0 +1,20 @@
533c21
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
533c21
+  <synapse id="0">
533c21
+    <action_set>
533c21
+      <rsc_op id="5" operation="start" operation_key="dummy-2_start_0" on_node="cluster02" on_node_uuid="2">
533c21
+        <primitive id="dummy-2" class="ocf" provider="pacemaker" type="Dummy"/>
533c21
+        <attributes CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" />
533c21
+      </rsc_op>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+  <synapse id="1">
533c21
+    <action_set>
533c21
+      <rsc_op id="2" operation="stop" operation_key="dummy-3_stop_0" on_node="cluster01" on_node_uuid="1">
533c21
+        <primitive id="dummy-3" class="ocf" provider="pacemaker" type="Dummy"/>
533c21
+        <attributes CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" />
533c21
+      </rsc_op>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+</transition_graph>
533c21
diff --git a/cts/scheduler/scores/failed-probe-clone.scores b/cts/scheduler/scores/failed-probe-clone.scores
533c21
new file mode 100644
533c21
index 0000000000..7418b7f153
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/scores/failed-probe-clone.scores
533c21
@@ -0,0 +1,33 @@
533c21
+
533c21
+pcmk__clone_allocate: ping-1-clone allocation score on cluster01: -INFINITY
533c21
+pcmk__clone_allocate: ping-1-clone allocation score on cluster02: -INFINITY
533c21
+pcmk__clone_allocate: ping-1:0 allocation score on cluster01: -INFINITY
533c21
+pcmk__clone_allocate: ping-1:0 allocation score on cluster02: -INFINITY
533c21
+pcmk__clone_allocate: ping-1:1 allocation score on cluster01: -INFINITY
533c21
+pcmk__clone_allocate: ping-1:1 allocation score on cluster02: -INFINITY
533c21
+pcmk__clone_allocate: ping-2-clone allocation score on cluster01: -INFINITY
533c21
+pcmk__clone_allocate: ping-2-clone allocation score on cluster02: 0
533c21
+pcmk__clone_allocate: ping-2:0 allocation score on cluster01: -INFINITY
533c21
+pcmk__clone_allocate: ping-2:0 allocation score on cluster02: 0
533c21
+pcmk__clone_allocate: ping-2:1 allocation score on cluster01: -INFINITY
533c21
+pcmk__clone_allocate: ping-2:1 allocation score on cluster02: 0
533c21
+pcmk__clone_allocate: ping-3-clone allocation score on cluster01: -INFINITY
533c21
+pcmk__clone_allocate: ping-3-clone allocation score on cluster02: -INFINITY
533c21
+pcmk__clone_allocate: ping-3:0 allocation score on cluster01: -INFINITY
533c21
+pcmk__clone_allocate: ping-3:0 allocation score on cluster02: -INFINITY
533c21
+pcmk__clone_allocate: ping-3:1 allocation score on cluster01: -INFINITY
533c21
+pcmk__clone_allocate: ping-3:1 allocation score on cluster02: -INFINITY
533c21
+pcmk__native_allocate: Fencing allocation score on cluster01: 0
533c21
+pcmk__native_allocate: Fencing allocation score on cluster02: 0
533c21
+pcmk__native_allocate: ping-1:0 allocation score on cluster01: -INFINITY
533c21
+pcmk__native_allocate: ping-1:0 allocation score on cluster02: -INFINITY
533c21
+pcmk__native_allocate: ping-1:1 allocation score on cluster01: -INFINITY
533c21
+pcmk__native_allocate: ping-1:1 allocation score on cluster02: -INFINITY
533c21
+pcmk__native_allocate: ping-2:0 allocation score on cluster01: -INFINITY
533c21
+pcmk__native_allocate: ping-2:0 allocation score on cluster02: 0
533c21
+pcmk__native_allocate: ping-2:1 allocation score on cluster01: -INFINITY
533c21
+pcmk__native_allocate: ping-2:1 allocation score on cluster02: -INFINITY
533c21
+pcmk__native_allocate: ping-3:0 allocation score on cluster01: -INFINITY
533c21
+pcmk__native_allocate: ping-3:0 allocation score on cluster02: -INFINITY
533c21
+pcmk__native_allocate: ping-3:1 allocation score on cluster01: -INFINITY
533c21
+pcmk__native_allocate: ping-3:1 allocation score on cluster02: -INFINITY
533c21
diff --git a/cts/scheduler/scores/failed-probe-primitive.scores b/cts/scheduler/scores/failed-probe-primitive.scores
533c21
new file mode 100644
533c21
index 0000000000..f313029451
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/scores/failed-probe-primitive.scores
533c21
@@ -0,0 +1,9 @@
533c21
+
533c21
+pcmk__native_allocate: Fencing allocation score on cluster01: 0
533c21
+pcmk__native_allocate: Fencing allocation score on cluster02: 0
533c21
+pcmk__native_allocate: dummy-1 allocation score on cluster01: -INFINITY
533c21
+pcmk__native_allocate: dummy-1 allocation score on cluster02: -INFINITY
533c21
+pcmk__native_allocate: dummy-2 allocation score on cluster01: -INFINITY
533c21
+pcmk__native_allocate: dummy-2 allocation score on cluster02: 0
533c21
+pcmk__native_allocate: dummy-3 allocation score on cluster01: -INFINITY
533c21
+pcmk__native_allocate: dummy-3 allocation score on cluster02: -INFINITY
533c21
diff --git a/cts/scheduler/summary/failed-probe-clone.summary b/cts/scheduler/summary/failed-probe-clone.summary
533c21
new file mode 100644
533c21
index 0000000000..ca15c302aa
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/summary/failed-probe-clone.summary
533c21
@@ -0,0 +1,46 @@
533c21
+Current cluster status:
533c21
+  * Node List:
533c21
+    * Online: [ cluster01 cluster02 ]
533c21
+
533c21
+  * Full List of Resources:
533c21
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
+    * Clone Set: ping-1-clone [ping-1]:
533c21
+      * Stopped: [ cluster01 cluster02 ]
533c21
+    * Clone Set: ping-2-clone [ping-2]:
533c21
+      * Stopped: [ cluster01 cluster02 ]
533c21
+    * Clone Set: ping-3-clone [ping-3]:
533c21
+      * ping-3	(ocf:pacemaker:ping):	 FAILED cluster01
533c21
+      * Stopped: [ cluster02 ]
533c21
+
533c21
+Transition Summary:
533c21
+  * Start      ping-2:0     ( cluster02 )
533c21
+  * Stop       ping-3:0     ( cluster01 )  due to node availability
533c21
+
533c21
+Executing Cluster Transition:
533c21
+  * Cluster action:  clear_failcount for ping-1 on cluster02
533c21
+  * Cluster action:  clear_failcount for ping-1 on cluster01
533c21
+  * Cluster action:  clear_failcount for ping-2 on cluster02
533c21
+  * Cluster action:  clear_failcount for ping-2 on cluster01
533c21
+  * Pseudo action:   ping-2-clone_start_0
533c21
+  * Cluster action:  clear_failcount for ping-3 on cluster01
533c21
+  * Cluster action:  clear_failcount for ping-3 on cluster02
533c21
+  * Pseudo action:   ping-3-clone_stop_0
533c21
+  * Resource action: ping-2          start on cluster02
533c21
+  * Pseudo action:   ping-2-clone_running_0
533c21
+  * Resource action: ping-3          stop on cluster01
533c21
+  * Pseudo action:   ping-3-clone_stopped_0
533c21
+  * Resource action: ping-2          monitor=10000 on cluster02
533c21
+
533c21
+Revised Cluster Status:
533c21
+  * Node List:
533c21
+    * Online: [ cluster01 cluster02 ]
533c21
+
533c21
+  * Full List of Resources:
533c21
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
+    * Clone Set: ping-1-clone [ping-1]:
533c21
+      * Stopped: [ cluster01 cluster02 ]
533c21
+    * Clone Set: ping-2-clone [ping-2]:
533c21
+      * Started: [ cluster02 ]
533c21
+      * Stopped: [ cluster01 ]
533c21
+    * Clone Set: ping-3-clone [ping-3]:
533c21
+      * Stopped: [ cluster01 cluster02 ]
533c21
diff --git a/cts/scheduler/summary/failed-probe-primitive.summary b/cts/scheduler/summary/failed-probe-primitive.summary
533c21
new file mode 100644
533c21
index 0000000000..a634e7f00b
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/summary/failed-probe-primitive.summary
533c21
@@ -0,0 +1,27 @@
533c21
+Current cluster status:
533c21
+  * Node List:
533c21
+    * Online: [ cluster01 cluster02 ]
533c21
+
533c21
+  * Full List of Resources:
533c21
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
+    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped
533c21
+    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped
533c21
+    * dummy-3	(ocf:pacemaker:Dummy):	 FAILED cluster01
533c21
+
533c21
+Transition Summary:
533c21
+  * Start      dummy-2     ( cluster02 )
533c21
+  * Stop       dummy-3     ( cluster01 )  due to node availability
533c21
+
533c21
+Executing Cluster Transition:
533c21
+  * Resource action: dummy-2         start on cluster02
533c21
+  * Resource action: dummy-3         stop on cluster01
533c21
+
533c21
+Revised Cluster Status:
533c21
+  * Node List:
533c21
+    * Online: [ cluster01 cluster02 ]
533c21
+
533c21
+  * Full List of Resources:
533c21
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
+    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped
533c21
+    * dummy-2	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
+    * dummy-3	(ocf:pacemaker:Dummy):	 Stopped
533c21
diff --git a/cts/scheduler/xml/failed-probe-clone.xml b/cts/scheduler/xml/failed-probe-clone.xml
533c21
new file mode 100644
533c21
index 0000000000..f677585bab
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/xml/failed-probe-clone.xml
533c21
@@ -0,0 +1,110 @@
533c21
+<cib crm_feature_set="3.3.0" validate-with="pacemaker-3.3" epoch="1" num_updates="37" admin_epoch="1" cib-last-written="Tue May  5 12:04:36 2020" update-origin="cluster01" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2">
533c21
+  <configuration>
533c21
+    <crm_config>
533c21
+      <cluster_property_set id="cib-bootstrap-options">
533c21
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
533c21
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-1.e97f9675f.git.el7-e97f9675f"/>
533c21
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
533c21
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="test-cluster"/>
533c21
+        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
533c21
+        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
533c21
+      </cluster_property_set>
533c21
+    </crm_config>
533c21
+    <nodes>
533c21
+      <node id="1" uname="cluster01"/>
533c21
+      <node id="2" uname="cluster02"/>
533c21
+    </nodes>
533c21
+    <resources>
533c21
+      <primitive class="stonith" id="Fencing" type="fence_xvm">
533c21
+        <instance_attributes id="Fencing-instance_attributes">
533c21
+          <nvpair id="Fencing-instance_attributes-ip_family" name="ip_family" value="ipv4"/>
533c21
+        </instance_attributes>
533c21
+        <operations>
533c21
+          <op id="Fencing-monitor-interval-60s" interval="60s" name="monitor"/>
533c21
+        </operations>
533c21
+      </primitive>
533c21
+      <clone id="ping-1-clone">
533c21
+        <primitive class="ocf" id="ping-1" provider="pacemaker" type="ping">
533c21
+          <instance_attributes id="ping-1-instance_attributes">
533c21
+            <nvpair id="ping-1-instance_attributes-dampen" name="dampen" value="5s"/>
533c21
+            <nvpair id="ping-1-instance_attributes-host_list" name="host_list" value="192.168.122.1"/>
533c21
+            <nvpair id="ping-1-instance_attributes-multiplier" name="multiplier" value="1000"/>
533c21
+          </instance_attributes>
533c21
+          <operations>
533c21
+            <op id="ping-1-monitor-interval-10s" interval="10s" name="monitor" timeout="60s"/>
533c21
+            <op id="ping-1-start-interval-0s" interval="0s" name="start" timeout="60s"/>
533c21
+            <op id="ping-1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
533c21
+          </operations>
533c21
+        </primitive>
533c21
+      </clone>
533c21
+      <clone id="ping-2-clone">
533c21
+        <primitive class="ocf" id="ping-2" provider="pacemaker" type="ping">
533c21
+          <instance_attributes id="ping-2-instance_attributes">
533c21
+            <nvpair id="ping-2-instance_attributes-dampen" name="dampen" value="5s"/>
533c21
+            <nvpair id="ping-2-instance_attributes-host_list" name="host_list" value="192.168.122.2"/>
533c21
+            <nvpair id="ping-2-instance_attributes-multiplier" name="multiplier" value="1000"/>
533c21
+          </instance_attributes>
533c21
+          <operations>
533c21
+            <op id="ping-2-monitor-interval-10s" interval="10s" name="monitor" timeout="60s"/>
533c21
+            <op id="ping-2-start-interval-0s" interval="0s" name="start" timeout="60s"/>
533c21
+            <op id="ping-2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
533c21
+          </operations>
533c21
+        </primitive>
533c21
+      </clone>
533c21
+      <clone id="ping-3-clone">
533c21
+        <primitive class="ocf" id="ping-3" provider="pacemaker" type="ping">
533c21
+          <instance_attributes id="ping-3-instance_attributes">
533c21
+            <nvpair id="ping-3-instance_attributes-dampen" name="dampen" value="5s"/>
533c21
+            <nvpair id="ping-3-instance_attributes-host_list" name="host_list" value="192.168.122.3"/>
533c21
+            <nvpair id="ping-3-instance_attributes-multiplier" name="multiplier" value="1000"/>
533c21
+          </instance_attributes>
533c21
+          <operations>
533c21
+            <op id="ping-3-monitor-interval-10s" interval="10s" name="monitor" timeout="60s"/>
533c21
+            <op id="ping-3-start-interval-0s" interval="0s" name="start" timeout="60s"/>
533c21
+            <op id="ping-3-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
533c21
+          </operations>
533c21
+        </primitive>
533c21
+      </clone>
533c21
+    </resources>
533c21
+    <constraints/>
533c21
+  </configuration>
533c21
+  <status>
533c21
+    <node_state id="1" uname="cluster01" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
533c21
+      <lrm id="1">
533c21
+        <lrm_resources>
533c21
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
533c21
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="3" rc-code="0" op-status="0" interval="0" last-rc-change="1588951272" exec-time="36" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
533c21
+            <lrm_rsc_op id="Fencing_monitor_60000" operation_key="Fencing_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="4" rc-code="0" op-status="0" interval="60000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="a88218bb6c7dc47e6586fc75fc2a8d69"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="ping-1" class="ocf" provider="pacemaker" type="ping">
533c21
+            <lrm_rsc_op id="ping-1_last_failure_0" operation_key="ping-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="5" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="ping-2" class="ocf" provider="pacemaker" type="ping">
533c21
+            <lrm_rsc_op id="ping-2_last_failure_0" operation_key="ping-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="6" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="ping-3" class="ocf" provider="pacemaker" type="ping">
533c21
+            <lrm_rsc_op id="ping-3_last_failure_0" operation_key="ping-3_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="9:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:4;9:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="9" rc-code="4" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+        </lrm_resources>
533c21
+      </lrm>
533c21
+    </node_state>
533c21
+    <node_state id="2" uname="cluster02" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
533c21
+      <lrm id="2">
533c21
+        <lrm_resources>
533c21
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
533c21
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:7;1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1588951263" exec-time="3" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="ping-1" class="ocf" provider="pacemaker" type="ping">
533c21
+            <lrm_rsc_op id="ping-1_last_failure_0" operation_key="ping-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="ping-2" class="ocf" provider="pacemaker" type="ping">
533c21
+            <lrm_rsc_op id="ping-2_last_failure_0" operation_key="ping-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="7:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;7:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="7" rc-code="7" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="ping-3" class="ocf" provider="pacemaker" type="ping">
533c21
+            <lrm_rsc_op id="ping-3_last_failure_0" operation_key="ping-3_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="8:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;8:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="8" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+        </lrm_resources>
533c21
+      </lrm>
533c21
+    </node_state>
533c21
+  </status>
533c21
+</cib>
533c21
diff --git a/cts/scheduler/xml/failed-probe-primitive.xml b/cts/scheduler/xml/failed-probe-primitive.xml
533c21
new file mode 100644
533c21
index 0000000000..0c2f6416f5
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/xml/failed-probe-primitive.xml
533c21
@@ -0,0 +1,71 @@
533c21
+<cib crm_feature_set="3.3.0" validate-with="pacemaker-3.3" epoch="1" num_updates="37" admin_epoch="1" cib-last-written="Tue May  5 12:04:36 2020" update-origin="cluster01" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2">
533c21
+  <configuration>
533c21
+    <crm_config>
533c21
+      <cluster_property_set id="cib-bootstrap-options">
533c21
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
533c21
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-1.e97f9675f.git.el7-e97f9675f"/>
533c21
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
533c21
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="test-cluster"/>
533c21
+        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
533c21
+        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
533c21
+      </cluster_property_set>
533c21
+    </crm_config>
533c21
+    <nodes>
533c21
+      <node id="1" uname="cluster01"/>
533c21
+      <node id="2" uname="cluster02"/>
533c21
+    </nodes>
533c21
+    <resources>
533c21
+      <primitive class="stonith" id="Fencing" type="fence_xvm">
533c21
+        <instance_attributes id="Fencing-instance_attributes">
533c21
+          <nvpair id="Fencing-instance_attributes-ip_family" name="ip_family" value="ipv4"/>
533c21
+        </instance_attributes>
533c21
+        <operations>
533c21
+          <op id="Fencing-monitor-interval-60s" interval="60s" name="monitor"/>
533c21
+        </operations>
533c21
+      </primitive>
533c21
+      <primitive class="ocf" id="dummy-1" provider="pacemaker" type="Dummy"/>
533c21
+      <primitive class="ocf" id="dummy-2" provider="pacemaker" type="Dummy"/>
533c21
+      <primitive class="ocf" id="dummy-3" provider="pacemaker" type="Dummy"/>
533c21
+    </resources>
533c21
+    <constraints/>
533c21
+  </configuration>
533c21
+  <status>
533c21
+    <node_state id="1" uname="cluster01" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
533c21
+      <lrm id="1">
533c21
+        <lrm_resources>
533c21
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
533c21
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="3" rc-code="0" op-status="0" interval="0" last-rc-change="1588951272" exec-time="36" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
533c21
+            <lrm_rsc_op id="Fencing_monitor_60000" operation_key="Fencing_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="4" rc-code="0" op-status="0" interval="60000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="a88218bb6c7dc47e6586fc75fc2a8d69"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-1_last_failure_0" operation_key="dummy-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="5" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;6:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="6" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="dummy-3" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-3_last_failure_0" operation_key="dummy-3_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="9:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:4;9:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="9" rc-code="4" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+        </lrm_resources>
533c21
+      </lrm>
533c21
+    </node_state>
533c21
+    <node_state id="2" uname="cluster02" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
533c21
+      <lrm id="2">
533c21
+        <lrm_resources>
533c21
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
533c21
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:7;1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1588951263" exec-time="3" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-1_last_failure_0" operation_key="dummy-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="7:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;7:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="7" rc-code="7" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="dummy-3" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-3_last_failure_0" operation_key="dummy-3_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="8:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;8:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="8" rc-code="5" op-status="0" interval="0" last-rc-change="1599063458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+        </lrm_resources>
533c21
+      </lrm>
533c21
+    </node_state>
533c21
+  </status>
533c21
+</cib>
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 271d50e7d6b0ee5ef670b571c6d7aae9272b75ad Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Thu, 11 Nov 2021 13:57:05 -0500
533c21
Subject: [PATCH 13/21] Feature: scheduler: Don't output failed resource
533c21
 probes...
533c21
533c21
in the crm_mon "Failed Resource Actions" section.  It is expected that
533c21
these one-off probes will fail, in which case displaying them in that
533c21
section can just come across as confusing to the user.
533c21
533c21
And update the crm_mon test output to account for these changes.
533c21
533c21
See: rhbz#1506372
533c21
---
533c21
 cts/cli/regression.crm_mon.exp | 20 --------------------
533c21
 lib/pengine/pe_output.c        |  4 ++++
533c21
 2 files changed, 4 insertions(+), 20 deletions(-)
533c21
533c21
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
533c21
index d7b9d98e2c..b1643f8b29 100644
533c21
--- a/cts/cli/regression.crm_mon.exp
533c21
+++ b/cts/cli/regression.crm_mon.exp
533c21
@@ -3498,10 +3498,6 @@ Active Resources:
533c21
 
533c21
 Failed Resource Actions:
533c21
   * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
-  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
533c21
-  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
533c21
-  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
533c21
-  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
533c21
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources
533c21
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
533c21
@@ -3646,10 +3642,6 @@ Failed Resource Actions:
533c21
   </node_history>
533c21
   <failures>
533c21
     <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="33" interval="0" task="monitor"/>
533c21
-    <failure op_key="dummy-4_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="0" interval="0" task="monitor"/>
533c21
-    <failure op_key="smart-mon_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="9" status="complete" last-rc-change="2021-11-09 15:38:55 -05:00" queued="0" exec="33" interval="0" task="monitor"/>
533c21
-    <failure op_key="ping_monitor_0" node="cluster02" exitstatus="not installed" exitreason="" exitcode="5" call="6" status="complete" last-rc-change="2021-11-18 13:11:42 -05:00" queued="0" exec="0" interval="0" task="monitor"/>
533c21
-    <failure op_key="httpd_monitor_0" node="httpd-bundle-1" exitstatus="invalid parameter" exitreason="" exitcode="2" call="1" status="complete" last-rc-change="2020-05-27 15:43:09 -04:00" queued="0" exec="0" interval="0" task="monitor"/>
533c21
   </failures>
533c21
   <status code="0" message="OK"/>
533c21
 </pacemaker-result>
533c21
@@ -3693,10 +3685,6 @@ Full List of Resources:
533c21
 
533c21
 Failed Resource Actions:
533c21
   * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
-  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
533c21
-  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
533c21
-  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
533c21
-  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
533c21
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
533c21
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
533c21
@@ -3784,10 +3772,6 @@ Operations:
533c21
 
533c21
 Failed Resource Actions:
533c21
   * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
-  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
533c21
-  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
533c21
-  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
533c21
-  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
533c21
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
533c21
@@ -3959,10 +3943,6 @@ Operations:
533c21
 
533c21
 Failed Resource Actions:
533c21
   * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
-  * dummy-4_monitor_0 on cluster02 'not installed' (5): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=0ms
533c21
-  * smart-mon_monitor_0 on cluster02 'not installed' (5): call=9, status='complete', last-rc-change='Tue Nov  9 15:38:55 2021', queued=0ms, exec=33ms
533c21
-  * ping_monitor_0 on cluster02 'not installed' (5): call=6, status='complete', last-rc-change='Thu Nov 18 13:11:42 2021', queued=0ms, exec=0ms
533c21
-  * httpd_monitor_0 on httpd-bundle-1 'invalid parameter' (2): call=1, status='complete', last-rc-change='Wed May 27 15:43:09 2020', queued=0ms, exec=0ms
533c21
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
533c21
diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
533c21
index 715e001d51..84684598dd 100644
533c21
--- a/lib/pengine/pe_output.c
533c21
+++ b/lib/pengine/pe_output.c
533c21
@@ -1370,6 +1370,10 @@ failed_action_list(pcmk__output_t *out, va_list args) {
533c21
             continue;
533c21
         }
533c21
 
533c21
+        if (pcmk_xe_mask_probe_failure(xml_op)) {
533c21
+            continue;
533c21
+        }
533c21
+
533c21
         id = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
533c21
         if (parse_op_key(id ? id : ID(xml_op), &rsc, NULL, NULL) == FALSE) {
533c21
             continue;
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 90f641b9223c64701d494297ce3dd3382365acb8 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Tue, 9 Nov 2021 10:11:19 -0500
533c21
Subject: [PATCH 14/21] Feature: scheduler: Add a function for finding a failed
533c21
 probe action...
533c21
533c21
for a given resource ID.  Optionally, a node ID can also be given to
533c21
restrict the failed probe action to one run on the given node.
533c21
Otherwise, just the first failed probe action for the resource ID will
533c21
be returned.
533c21
533c21
See: rhbz#1506372
533c21
---
533c21
 include/crm/pengine/internal.h |  2 ++
533c21
 lib/pengine/utils.c            | 42 ++++++++++++++++++++++++++++++++++
533c21
 2 files changed, 44 insertions(+)
533c21
533c21
diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
533c21
index 8c8fbaca90..58dd2e8727 100644
533c21
--- a/include/crm/pengine/internal.h
533c21
+++ b/include/crm/pengine/internal.h
533c21
@@ -574,4 +574,6 @@ gboolean pe__clone_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean che
533c21
 gboolean pe__group_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent);
533c21
 gboolean pe__native_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent);
533c21
 
533c21
+xmlNode *pe__failed_probe_for_rsc(pe_resource_t *rsc, const char *name);
533c21
+
533c21
 #endif
533c21
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
533c21
index 07753e173a..3151f0120b 100644
533c21
--- a/lib/pengine/utils.c
533c21
+++ b/lib/pengine/utils.c
533c21
@@ -2569,3 +2569,45 @@ pe__build_rsc_list(pe_working_set_t *data_set, const char *s) {
533c21
 
533c21
     return resources;
533c21
 }
533c21
+
533c21
+xmlNode *
533c21
+pe__failed_probe_for_rsc(pe_resource_t *rsc, const char *name)
533c21
+{
533c21
+    const char *rsc_id = rsc->id;
533c21
+
533c21
+    for (xmlNode *xml_op = pcmk__xml_first_child(rsc->cluster->failed); xml_op != NULL;
533c21
+         xml_op = pcmk__xml_next(xml_op)) {
533c21
+        const char *value = NULL;
533c21
+        char *op_id = NULL;
533c21
+
533c21
+        /* This resource operation is not a failed probe. */
533c21
+        if (!pcmk_xe_mask_probe_failure(xml_op)) {
533c21
+            continue;
533c21
+        }
533c21
+
533c21
+        /* This resource operation was not run on the given node.  Note that if name is
533c21
+         * NULL, this will always succeed.
533c21
+         */
533c21
+        value = crm_element_value(xml_op, XML_LRM_ATTR_TARGET);
533c21
+        if (value == NULL || !pcmk__str_eq(value, name, pcmk__str_casei|pcmk__str_null_matches)) {
533c21
+            continue;
533c21
+        }
533c21
+
533c21
+        /* This resource operation has no operation_key. */
533c21
+        value = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
533c21
+        if (!parse_op_key(value ? value : ID(xml_op), &op_id, NULL, NULL)) {
533c21
+            continue;
533c21
+        }
533c21
+
533c21
+        /* This resource operation's ID does not match the rsc_id we are looking for. */
533c21
+        if (!pcmk__str_eq(op_id, rsc_id, pcmk__str_none)) {
533c21
+            free(op_id);
533c21
+            continue;
533c21
+        }
533c21
+
533c21
+        free(op_id);
533c21
+        return xml_op;
533c21
+    }
533c21
+
533c21
+    return NULL;
533c21
+}
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 2ad9774fe994554243078b131799fed0d1a6dffd Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Tue, 9 Nov 2021 15:43:24 -0500
533c21
Subject: [PATCH 15/21] Feature: scheduler: Display the reason why a native rsc
533c21
 probe failed.
533c21
533c21
If inactive resources are being shown, add an extra blurb of text to any
533c21
stopped resources that have a failed probe action indicating why the
533c21
probe failed.
533c21
533c21
And then add a new primitive resource to crm_mon-partial.xml with a
533c21
failed probe operation and update the expected test output.
533c21
533c21
See: rhbz#1506372
533c21
---
533c21
 cts/cli/regression.crm_mon.exp                        | 10 +++++-----
533c21
 cts/scheduler/summary/failed-probe-primitive.summary  |  8 ++++----
533c21
 cts/scheduler/summary/multiply-active-stonith.summary |  2 +-
533c21
 lib/pengine/native.c                                  | 11 +++++++++++
533c21
 4 files changed, 21 insertions(+), 10 deletions(-)
533c21
533c21
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
533c21
index b1643f8b29..4333caa11c 100644
533c21
--- a/cts/cli/regression.crm_mon.exp
533c21
+++ b/cts/cli/regression.crm_mon.exp
533c21
@@ -3680,8 +3680,8 @@ Full List of Resources:
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
     * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
533c21
-    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped
533c21
-  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
+    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
533c21
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped (not installed) 
533c21
 
533c21
 Failed Resource Actions:
533c21
   * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
@@ -3811,7 +3811,7 @@ Full List of Resources:
533c21
     * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
     * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (disabled)
533c21
-    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped
533c21
+    * dummy-4	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
533c21
 =#=#=#= End test: Text output of partially active group, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active group, with inactive resources
533c21
 =#=#=#= Begin test: Text output of active member of partially active group =#=#=#=
533c21
@@ -3889,7 +3889,7 @@ Inactive Resources:
533c21
     * ping	(ocf:pacemaker:ping):	 Stopped
533c21
   * Resource Group: partially-active-group:
533c21
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
533c21
-  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped (not installed) 
533c21
 
533c21
 Node Attributes:
533c21
   * Node: cluster01 (1):
533c21
@@ -3963,7 +3963,7 @@ Full List of Resources:
533c21
   * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
     * httpd-bundle-1 (192.168.122.132)	(ocf:heartbeat:apache):	 FAILED cluster01
533c21
-  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped
533c21
+  * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped (not installed) 
533c21
 =#=#=#= End test: Text output of partially active resources, with inactive resources, filtered by node - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources, filtered by node
533c21
 =#=#=#= Begin test: Text output of partially active resources, filtered by node =#=#=#=
533c21
diff --git a/cts/scheduler/summary/failed-probe-primitive.summary b/cts/scheduler/summary/failed-probe-primitive.summary
533c21
index a634e7f00b..ea8edae494 100644
533c21
--- a/cts/scheduler/summary/failed-probe-primitive.summary
533c21
+++ b/cts/scheduler/summary/failed-probe-primitive.summary
533c21
@@ -4,8 +4,8 @@ Current cluster status:
533c21
 
533c21
   * Full List of Resources:
533c21
     * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
-    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped
533c21
-    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped
533c21
+    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
533c21
+    * dummy-2	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
533c21
     * dummy-3	(ocf:pacemaker:Dummy):	 FAILED cluster01
533c21
 
533c21
 Transition Summary:
533c21
@@ -22,6 +22,6 @@ Revised Cluster Status:
533c21
 
533c21
   * Full List of Resources:
533c21
     * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
-    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped
533c21
+    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
-    * dummy-3	(ocf:pacemaker:Dummy):	 Stopped
533c21
+    * dummy-3	(ocf:pacemaker:Dummy):	 Stopped (not installed) 
533c21
diff --git a/cts/scheduler/summary/multiply-active-stonith.summary b/cts/scheduler/summary/multiply-active-stonith.summary
533c21
index 8ce21d68ee..ec37de03b0 100644
533c21
--- a/cts/scheduler/summary/multiply-active-stonith.summary
533c21
+++ b/cts/scheduler/summary/multiply-active-stonith.summary
533c21
@@ -25,4 +25,4 @@ Revised Cluster Status:
533c21
 
533c21
   * Full List of Resources:
533c21
     * fencer	(stonith:fence_ipmilan):	 Started node3
533c21
-    * rsc1	(lsb:rsc1):	 Stopped
533c21
+    * rsc1	(lsb:rsc1):	 Stopped (not installed) 
533c21
diff --git a/lib/pengine/native.c b/lib/pengine/native.c
533c21
index 36121c527f..a95c90c09a 100644
533c21
--- a/lib/pengine/native.c
533c21
+++ b/lib/pengine/native.c
533c21
@@ -599,6 +599,17 @@ pcmk__native_output_string(pe_resource_t *rsc, const char *name, pe_node_t *node
533c21
         g_string_append_printf(outstr, " %s", node->details->uname);
533c21
     }
533c21
 
533c21
+    // Failed probe operation
533c21
+    if (native_displayable_role(rsc) == RSC_ROLE_STOPPED) {
533c21
+        xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node ? node->details->uname : NULL);
533c21
+        if (probe_op != NULL) {
533c21
+            int rc;
533c21
+
533c21
+            pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
533c21
+            g_string_append_printf(outstr, " (%s) ", services_ocf_exitcode_str(rc));
533c21
+        }
533c21
+    }
533c21
+
533c21
     // Flags, as: (<flag> [...])
533c21
     if (node && !(node->details->online) && node->details->unclean) {
533c21
         have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From b9ca2e834ee01b35c03f153438ef8828b609fb38 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Thu, 18 Nov 2021 10:41:42 -0500
533c21
Subject: [PATCH 16/21] Refactor: scheduler: Rearrange pe__clone_default.
533c21
533c21
Instead of the single stopped list, maintain a hash table where the keys
533c21
are nodes and the values are the status of the node.  For now, this is
533c21
just "Stopped" or "Stopped (disabled)" but in the future will be
533c21
expanded to cover failed probe operations.
533c21
---
533c21
 lib/pengine/clone.c | 103 +++++++++++++++++++++++++++++++++++---------
533c21
 1 file changed, 82 insertions(+), 21 deletions(-)
533c21
533c21
diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
533c21
index 5569c6b6e9..58fb24d24e 100644
533c21
--- a/lib/pengine/clone.c
533c21
+++ b/lib/pengine/clone.c
533c21
@@ -28,6 +28,55 @@
533c21
 #define UNPROMOTED_INSTANCES RSC_ROLE_UNPROMOTED_S
533c21
 #endif
533c21
 
533c21
+static GList *
533c21
+sorted_hash_table_values(GHashTable *table)
533c21
+{
533c21
+    GList *retval = NULL;
533c21
+    GHashTableIter iter;
533c21
+    gpointer key, value;
533c21
+
533c21
+    g_hash_table_iter_init(&iter, table);
533c21
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
533c21
+        if (!g_list_find_custom(retval, value, (GCompareFunc) strcmp)) {
533c21
+            retval = g_list_prepend(retval, (char *) value);
533c21
+        }
533c21
+    }
533c21
+
533c21
+    retval = g_list_sort(retval, (GCompareFunc) strcmp);
533c21
+    return retval;
533c21
+}
533c21
+
533c21
+static GList *
533c21
+nodes_with_status(GHashTable *table, const char *status)
533c21
+{
533c21
+    GList *retval = NULL;
533c21
+    GHashTableIter iter;
533c21
+    gpointer key, value;
533c21
+
533c21
+    g_hash_table_iter_init(&iter, table);
533c21
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
533c21
+        if (!strcmp((char *) value, status)) {
533c21
+            retval = g_list_prepend(retval, key);
533c21
+        }
533c21
+    }
533c21
+
533c21
+    retval = g_list_sort(retval, (GCompareFunc) pcmk__numeric_strcasecmp);
533c21
+    return retval;
533c21
+}
533c21
+
533c21
+static char *
533c21
+node_list_to_str(GList *list)
533c21
+{
533c21
+    char *retval = NULL;
533c21
+    size_t len = 0;
533c21
+
533c21
+    for (GList *iter = list; iter != NULL; iter = iter->next) {
533c21
+        pcmk__add_word(&retval, &len, (char *) iter->data);
533c21
+    }
533c21
+
533c21
+    return retval;
533c21
+}
533c21
+
533c21
 static void
533c21
 clone_header(pcmk__output_t *out, int *rc, pe_resource_t *rsc, clone_variant_data_t *clone_data)
533c21
 {
533c21
@@ -710,10 +759,10 @@ pe__clone_default(pcmk__output_t *out, va_list args)
533c21
     GList *only_node = va_arg(args, GList *);
533c21
     GList *only_rsc = va_arg(args, GList *);
533c21
 
533c21
+    GHashTable *stopped = pcmk__strkey_table(free, free);
533c21
+
533c21
     char *list_text = NULL;
533c21
-    char *stopped_list = NULL;
533c21
     size_t list_text_len = 0;
533c21
-    size_t stopped_list_len = 0;
533c21
 
533c21
     GList *promoted_list = NULL;
533c21
     GList *started_list = NULL;
533c21
@@ -768,7 +817,7 @@ pe__clone_default(pcmk__output_t *out, va_list args)
533c21
             // List stopped instances when requested (except orphans)
533c21
             if (!pcmk_is_set(child_rsc->flags, pe_rsc_orphan)
533c21
                 && pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
533c21
-                pcmk__add_word(&stopped_list, &stopped_list_len, child_rsc->id);
533c21
+                g_hash_table_insert(stopped, strdup(child_rsc->id), strdup("Stopped"));
533c21
             }
533c21
 
533c21
         } else if (is_set_recursive(child_rsc, pe_rsc_orphan, TRUE)
533c21
@@ -822,7 +871,7 @@ pe__clone_default(pcmk__output_t *out, va_list args)
533c21
     }
533c21
 
533c21
     if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
533c21
-        free(stopped_list);
533c21
+        g_hash_table_destroy(stopped);
533c21
         PCMK__OUTPUT_LIST_FOOTER(out, rc);
533c21
         return pcmk_rc_ok;
533c21
     }
533c21
@@ -890,23 +939,15 @@ pe__clone_default(pcmk__output_t *out, va_list args)
533c21
     }
533c21
 
533c21
     if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
533c21
-        const char *state = "Stopped";
533c21
-        enum rsc_role_e role = configured_role(rsc);
533c21
-
533c21
-        if (role == RSC_ROLE_STOPPED) {
533c21
-            state = "Stopped (disabled)";
533c21
-        }
533c21
-
533c21
         if (!pcmk_is_set(rsc->flags, pe_rsc_unique)
533c21
             && (clone_data->clone_max > active_instances)) {
533c21
 
533c21
             GList *nIter;
533c21
             GList *list = g_hash_table_get_values(rsc->allowed_nodes);
533c21
 
533c21
-            /* Custom stopped list for non-unique clones */
533c21
-            free(stopped_list);
533c21
-            stopped_list = NULL;
533c21
-            stopped_list_len = 0;
533c21
+            /* Custom stopped table for non-unique clones */
533c21
+            g_hash_table_destroy(stopped);
533c21
+            stopped = pcmk__strkey_table(free, free);
533c21
 
533c21
             if (list == NULL) {
533c21
                 /* Clusters with symmetrical=false haven't calculated allowed_nodes yet
533c21
@@ -922,19 +963,39 @@ pe__clone_default(pcmk__output_t *out, va_list args)
533c21
                 if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
533c21
                     pcmk__str_in_list(node->details->uname, only_node,
533c21
                                       pcmk__str_star_matches|pcmk__str_casei)) {
533c21
-                    pcmk__add_word(&stopped_list, &stopped_list_len,
533c21
-                                   node->details->uname);
533c21
+                    const char *state = "Stopped";
533c21
+
533c21
+                    if (configured_role(rsc) == RSC_ROLE_STOPPED) {
533c21
+                        state = "Stopped (disabled)";
533c21
+                    }
533c21
+
533c21
+                    g_hash_table_insert(stopped, strdup(node->details->uname),
533c21
+                                        strdup(state));
533c21
                 }
533c21
             }
533c21
             g_list_free(list);
533c21
         }
533c21
 
533c21
-        if (stopped_list != NULL) {
533c21
+        if (g_hash_table_size(stopped) > 0) {
533c21
+            GList *list = sorted_hash_table_values(stopped);
533c21
+
533c21
             clone_header(out, &rc, rsc, clone_data);
533c21
 
533c21
-            out->list_item(out, NULL, "%s: [ %s ]", state, stopped_list);
533c21
-            free(stopped_list);
533c21
-            stopped_list_len = 0;
533c21
+            for (GList *status_iter = list; status_iter != NULL; status_iter = status_iter->next) {
533c21
+                const char *status = status_iter->data;
533c21
+                GList *nodes = nodes_with_status(stopped, status);
533c21
+                char *str = node_list_to_str(nodes);
533c21
+
533c21
+                if (str != NULL) {
533c21
+                    out->list_item(out, NULL, "%s: [ %s ]", status, str);
533c21
+                    free(str);
533c21
+                }
533c21
+
533c21
+                g_list_free(nodes);
533c21
+            }
533c21
+
533c21
+            g_list_free(list);
533c21
+            g_hash_table_destroy(stopped);
533c21
 
533c21
         /* If there are no instances of this clone (perhaps because there are no
533c21
          * nodes configured), simply output the clone header by itself.  This can
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From 0228a64cea412936fb8ee91b0f83f9800048d3ba Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Fri, 19 Nov 2021 10:06:18 -0500
533c21
Subject: [PATCH 17/21] Feature: scheduler: Display the reason why a clone rsc
533c21
 probe failed.
533c21
533c21
This is similar to the previous commit that adds reasons for primitive
533c21
resources.
533c21
533c21
See: rhbz#1506372
533c21
---
533c21
 cts/cli/regression.crm_mon.exp                |  8 +++----
533c21
 .../summary/failed-probe-clone.summary        | 14 +++++++------
533c21
 include/crm/pengine/internal.h                |  2 ++
533c21
 lib/pengine/clone.c                           | 21 +++++++++++++++++--
533c21
 lib/pengine/utils.c                           |  7 +++++++
533c21
 5 files changed, 40 insertions(+), 12 deletions(-)
533c21
533c21
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
533c21
index 4333caa11c..5688500ce5 100644
533c21
--- a/cts/cli/regression.crm_mon.exp
533c21
+++ b/cts/cli/regression.crm_mon.exp
533c21
@@ -3479,7 +3479,7 @@ Node List:
533c21
 Active Resources:
533c21
   * Clone Set: ping-clone [ping]:
533c21
     * ping	(ocf:pacemaker:ping):	 Started cluster01
533c21
-    * ping	(ocf:pacemaker:ping):	 Stopped
533c21
+    * ping	(ocf:pacemaker:ping):	 Stopped (not installed) 
533c21
   * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
     * Replica[0]
533c21
@@ -3663,7 +3663,7 @@ Node List:
533c21
 Full List of Resources:
533c21
   * Clone Set: ping-clone [ping]:
533c21
     * ping	(ocf:pacemaker:ping):	 Started cluster01
533c21
-    * ping	(ocf:pacemaker:ping):	 Stopped
533c21
+    * ping	(ocf:pacemaker:ping):	 Stopped (not installed) 
533c21
   * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
     * Replica[0]
533c21
@@ -3705,7 +3705,7 @@ Full List of Resources:
533c21
   * 1/1	(stonith:fence_xvm):	Active cluster01
533c21
   * Clone Set: ping-clone [ping]:
533c21
     * ping	(ocf:pacemaker:ping):	 Started cluster01
533c21
-    * ping	(ocf:pacemaker:ping):	 Stopped
533c21
+    * ping	(ocf:pacemaker:ping):	 Stopped (not installed) 
533c21
   * Container bundle set: httpd-bundle [pcmk:http]:
533c21
     * Replica[0]
533c21
       * httpd-bundle-ip-192.168.122.131	(ocf:heartbeat:IPaddr2):	 Started cluster02
533c21
@@ -3886,7 +3886,7 @@ Node List:
533c21
 Inactive Resources:
533c21
   * Clone Set: ping-clone [ping]:
533c21
     * ping	(ocf:pacemaker:ping):	 Started cluster01
533c21
-    * ping	(ocf:pacemaker:ping):	 Stopped
533c21
+    * ping	(ocf:pacemaker:ping):	 Stopped (not installed) 
533c21
   * Resource Group: partially-active-group:
533c21
     * 2/4	(ocf:pacemaker:Dummy):	Active cluster02
533c21
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped (not installed) 
533c21
diff --git a/cts/scheduler/summary/failed-probe-clone.summary b/cts/scheduler/summary/failed-probe-clone.summary
533c21
index ca15c302aa..febee14400 100644
533c21
--- a/cts/scheduler/summary/failed-probe-clone.summary
533c21
+++ b/cts/scheduler/summary/failed-probe-clone.summary
533c21
@@ -5,12 +5,13 @@ Current cluster status:
533c21
   * Full List of Resources:
533c21
     * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
     * Clone Set: ping-1-clone [ping-1]:
533c21
-      * Stopped: [ cluster01 cluster02 ]
533c21
+      * Stopped (not installed): [ cluster01 cluster02 ]
533c21
     * Clone Set: ping-2-clone [ping-2]:
533c21
-      * Stopped: [ cluster01 cluster02 ]
533c21
+      * Stopped: [ cluster02 ]
533c21
+      * Stopped (not installed): [ cluster01 ]
533c21
     * Clone Set: ping-3-clone [ping-3]:
533c21
       * ping-3	(ocf:pacemaker:ping):	 FAILED cluster01
533c21
-      * Stopped: [ cluster02 ]
533c21
+      * Stopped (not installed): [ cluster02 ]
533c21
 
533c21
 Transition Summary:
533c21
   * Start      ping-2:0     ( cluster02 )
533c21
@@ -38,9 +39,10 @@ Revised Cluster Status:
533c21
   * Full List of Resources:
533c21
     * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
     * Clone Set: ping-1-clone [ping-1]:
533c21
-      * Stopped: [ cluster01 cluster02 ]
533c21
+      * Stopped (not installed): [ cluster01 cluster02 ]
533c21
     * Clone Set: ping-2-clone [ping-2]:
533c21
       * Started: [ cluster02 ]
533c21
-      * Stopped: [ cluster01 ]
533c21
+      * Stopped (not installed): [ cluster01 ]
533c21
     * Clone Set: ping-3-clone [ping-3]:
533c21
-      * Stopped: [ cluster01 cluster02 ]
533c21
+      * Stopped: [ cluster01 ]
533c21
+      * Stopped (not installed): [ cluster02 ]
533c21
diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
533c21
index 58dd2e8727..2b20da6e5f 100644
533c21
--- a/include/crm/pengine/internal.h
533c21
+++ b/include/crm/pengine/internal.h
533c21
@@ -576,4 +576,6 @@ gboolean pe__native_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean ch
533c21
 
533c21
 xmlNode *pe__failed_probe_for_rsc(pe_resource_t *rsc, const char *name);
533c21
 
533c21
+const char *pe__clone_child_id(pe_resource_t *rsc);
533c21
+
533c21
 #endif
533c21
diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
533c21
index 58fb24d24e..ef4bdc0edf 100644
533c21
--- a/lib/pengine/clone.c
533c21
+++ b/lib/pengine/clone.c
533c21
@@ -963,14 +963,23 @@ pe__clone_default(pcmk__output_t *out, va_list args)
533c21
                 if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
533c21
                     pcmk__str_in_list(node->details->uname, only_node,
533c21
                                       pcmk__str_star_matches|pcmk__str_casei)) {
533c21
+                    xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node->details->uname);
533c21
                     const char *state = "Stopped";
533c21
 
533c21
                     if (configured_role(rsc) == RSC_ROLE_STOPPED) {
533c21
                         state = "Stopped (disabled)";
533c21
                     }
533c21
 
533c21
-                    g_hash_table_insert(stopped, strdup(node->details->uname),
533c21
-                                        strdup(state));
533c21
+                    if (probe_op != NULL) {
533c21
+                        int rc;
533c21
+
533c21
+                        pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
533c21
+                        g_hash_table_insert(stopped, strdup(node->details->uname),
533c21
+                                            crm_strdup_printf("Stopped (%s)", services_ocf_exitcode_str(rc)));
533c21
+                    } else {
533c21
+                        g_hash_table_insert(stopped, strdup(node->details->uname),
533c21
+                                            strdup(state));
533c21
+                    }
533c21
                 }
533c21
             }
533c21
             g_list_free(list);
533c21
@@ -1113,3 +1122,11 @@ pe__clone_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent
533c21
 
533c21
     return !passes;
533c21
 }
533c21
+
533c21
+const char *
533c21
+pe__clone_child_id(pe_resource_t *rsc)
533c21
+{
533c21
+    clone_variant_data_t *clone_data = NULL;
533c21
+    get_clone_variant_data(clone_data, rsc);
533c21
+    return ID(clone_data->xml_obj_child);
533c21
+}
533c21
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
533c21
index 3151f0120b..6c4f3b6971 100644
533c21
--- a/lib/pengine/utils.c
533c21
+++ b/lib/pengine/utils.c
533c21
@@ -2573,8 +2573,15 @@ pe__build_rsc_list(pe_working_set_t *data_set, const char *s) {
533c21
 xmlNode *
533c21
 pe__failed_probe_for_rsc(pe_resource_t *rsc, const char *name)
533c21
 {
533c21
+    pe_resource_t *parent = uber_parent(rsc);
533c21
     const char *rsc_id = rsc->id;
533c21
 
533c21
+    if (rsc->variant == pe_clone) {
533c21
+        rsc_id = pe__clone_child_id(rsc);
533c21
+    } else if (parent->variant == pe_clone) {
533c21
+        rsc_id = pe__clone_child_id(parent);
533c21
+    }
533c21
+
533c21
     for (xmlNode *xml_op = pcmk__xml_first_child(rsc->cluster->failed); xml_op != NULL;
533c21
          xml_op = pcmk__xml_next(xml_op)) {
533c21
         const char *value = NULL;
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From cf8b01da93fce87526617fefdcee6eb9f6ecdbd1 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Wed, 24 Nov 2021 10:57:05 -0500
533c21
Subject: [PATCH 18/21] Test: cts-cli: Update the last-rc-change sed
533c21
 expression.
533c21
533c21
This can now occur in both the XML output (where it's wrapped in double
533c21
quotes) and the text output (where it's wrapped in single quotes and
533c21
followed by a comma).  In addition, a plus or minus can occur in the
533c21
time string.
533c21
533c21
The "{0,1}" syntax takes the place of a "?" for marking the optional
533c21
comma.  In FreeBSD sed, "?" doesn't mean anything special.
533c21
---
533c21
 cts/cli/regression.crm_mon.exp | 12 ++++++------
533c21
 cts/cts-cli.in                 |  2 +-
533c21
 2 files changed, 7 insertions(+), 7 deletions(-)
533c21
533c21
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
533c21
index 5688500ce5..957758832d 100644
533c21
--- a/cts/cli/regression.crm_mon.exp
533c21
+++ b/cts/cli/regression.crm_mon.exp
533c21
@@ -3497,7 +3497,7 @@ Active Resources:
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
 
533c21
 Failed Resource Actions:
533c21
-  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', queued=0ms, exec=33ms
533c21
 =#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources
533c21
 =#=#=#= Begin test: XML output of partially active resources =#=#=#=
533c21
@@ -3641,7 +3641,7 @@ Failed Resource Actions:
533c21
     </node>
533c21
   </node_history>
533c21
   <failures>
533c21
-    <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" last-rc-change="2020-09-02 12:17:38 -04:00" queued="0" exec="33" interval="0" task="monitor"/>
533c21
+    <failure op_key="dummy-2_monitor_0" node="cluster02" exitstatus="unimplemented feature" exitreason="" exitcode="3" call="2" status="complete" queued="0" exec="33" interval="0" task="monitor"/>
533c21
   </failures>
533c21
   <status code="0" message="OK"/>
533c21
 </pacemaker-result>
533c21
@@ -3684,7 +3684,7 @@ Full List of Resources:
533c21
   * smart-mon	(ocf:pacemaker:HealthSMART):	 Stopped (not installed) 
533c21
 
533c21
 Failed Resource Actions:
533c21
-  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', queued=0ms, exec=33ms
533c21
 =#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of partially active resources, with inactive resources
533c21
 =#=#=#= Begin test: Complete brief text output, with inactive resources =#=#=#=
533c21
@@ -3771,7 +3771,7 @@ Operations:
533c21
       * (1) probe
533c21
 
533c21
 Failed Resource Actions:
533c21
-  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', queued=0ms, exec=33ms
533c21
 =#=#=#= End test: Complete brief text output, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active group =#=#=#=
533c21
@@ -3850,7 +3850,7 @@ Active Resources:
533c21
     * dummy-2	(ocf:pacemaker:Dummy):	 FAILED cluster02
533c21
 
533c21
 Failed Resource Actions:
533c21
-  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', queued=0ms, exec=33ms
533c21
 =#=#=#= End test: Text output of inactive member of partially active group - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Text output of inactive member of partially active group
533c21
 =#=#=#= Begin test: Complete brief text output grouped by node, with inactive resources =#=#=#=
533c21
@@ -3942,7 +3942,7 @@ Operations:
533c21
       * (1) probe
533c21
 
533c21
 Failed Resource Actions:
533c21
-  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', last-rc-change='Wed Sep  2 12:17:38 2020', queued=0ms, exec=33ms
533c21
+  * dummy-2_monitor_0 on cluster02 'unimplemented feature' (3): call=2, status='complete', queued=0ms, exec=33ms
533c21
 =#=#=#= End test: Complete brief text output grouped by node, with inactive resources - OK (0) =#=#=#=
533c21
 * Passed: crm_mon        - Complete brief text output grouped by node, with inactive resources
533c21
 =#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
533c21
diff --git a/cts/cts-cli.in b/cts/cts-cli.in
533c21
index 457816afab..72e9a1e912 100755
533c21
--- a/cts/cts-cli.in
533c21
+++ b/cts/cts-cli.in
533c21
@@ -1870,7 +1870,7 @@ for t in $tests; do
533c21
         -e 's/.*\(unpack_.*\)@.*\.c:[0-9][0-9]*)/\1/g' \
533c21
         -e 's/.*\(update_validation\)@.*\.c:[0-9][0-9]*)/\1/g' \
533c21
         -e 's/.*\(apply_upgrade\)@.*\.c:[0-9][0-9]*)/\1/g' \
533c21
-        -e 's/ last-rc-change=\"[A-Za-z0-9: ]*\"//'\
533c21
+        -e "s/ last-rc-change=['\"][-+A-Za-z0-9: ]*['\"],\{0,1\}//" \
533c21
         -e 's|^/tmp/cts-cli\.validity\.bad.xml\.[^:]*:|validity.bad.xml:|'\
533c21
         -e 's/^Entity: line [0-9][0-9]*: //'\
533c21
         -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From dea61f1b6507fbc978e040c1555384d8d7ffa9f3 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Wed, 1 Dec 2021 16:23:14 -0500
533c21
Subject: [PATCH 19/21] Fix: include: Bump feature set to 3.12.0.
533c21
533c21
This is for the scheduler handling changing regarding maskable probe
533c21
failures.
533c21
533c21
See: rhbz#1506372.
533c21
---
533c21
 include/crm/crm.h | 2 +-
533c21
 1 file changed, 1 insertion(+), 1 deletion(-)
533c21
533c21
diff --git a/include/crm/crm.h b/include/crm/crm.h
533c21
index 04d2324d75..16b35e9c55 100644
533c21
--- a/include/crm/crm.h
533c21
+++ b/include/crm/crm.h
533c21
@@ -66,7 +66,7 @@ extern "C" {
533c21
  * >=3.0.13: Fail counts include operation name and interval
533c21
  * >=3.2.0:  DC supports PCMK_EXEC_INVALID and PCMK_EXEC_NOT_CONNECTED
533c21
  */
533c21
-#  define CRM_FEATURE_SET		"3.11.0"
533c21
+#  define CRM_FEATURE_SET		"3.12.0"
533c21
 
533c21
 /* Pacemaker's CPG protocols use fixed-width binary fields for the sender and
533c21
  * recipient of a CPG message. This imposes an arbitrary limit on cluster node
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From fef2c61ef462c221809dc91467ea1e96d5478c74 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Mon, 6 Dec 2021 16:42:15 -0500
533c21
Subject: [PATCH 20/21] Feature: scheduler: Handle masked probes in the
533c21
 scheduler.
533c21
533c21
These probe operations get their rc/status codes mapped to not
533c21
running/done, but still ensures they end up in the list of failed
533c21
operations so tool output continues to display them properly.
533c21
533c21
Note that failures on bundled resources do not get masked.
533c21
533c21
There are no test case changes for this patch.
533c21
533c21
See: rhbz#1506372.
533c21
---
533c21
 lib/pengine/unpack.c | 42 +++++++++++++++++++++++++++++++++++++-----
533c21
 1 file changed, 37 insertions(+), 5 deletions(-)
533c21
533c21
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
533c21
index b659f319fb..f3583e97d8 100644
533c21
--- a/lib/pengine/unpack.c
533c21
+++ b/lib/pengine/unpack.c
533c21
@@ -3169,6 +3169,11 @@ remap_operation(xmlNode *xml_op, pe_resource_t *rsc, pe_node_t *node,
533c21
         }
533c21
     }
533c21
 
533c21
+    if (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
533c21
+        *status = PCMK_EXEC_DONE;
533c21
+        *rc = PCMK_OCF_NOT_RUNNING;
533c21
+    }
533c21
+
533c21
     /* If the executor reported an operation status of anything but done or
533c21
      * error, consider that final. But for done or error, we know better whether
533c21
      * it should be treated as a failure or not, because we know the expected
533c21
@@ -3567,12 +3572,12 @@ update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, c
533c21
     CRM_ASSERT(rsc);
533c21
     CRM_ASSERT(xml_op);
533c21
 
533c21
-    if (rc == PCMK_OCF_NOT_RUNNING) {
533c21
-        clear_past_failure = TRUE;
533c21
-
533c21
-    } else if (rc == PCMK_OCF_NOT_INSTALLED) {
533c21
+    if (rc == PCMK_OCF_NOT_INSTALLED || (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op))) {
533c21
         rsc->role = RSC_ROLE_STOPPED;
533c21
 
533c21
+    } else if (rc == PCMK_OCF_NOT_RUNNING) {
533c21
+        clear_past_failure = TRUE;
533c21
+
533c21
     } else if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)) {
533c21
         if (last_failure) {
533c21
             const char *op_key = get_op_key(xml_op);
533c21
@@ -3661,8 +3666,10 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
533c21
               pe_working_set_t *data_set)
533c21
 {
533c21
     int rc = 0;
533c21
+    int old_rc = 0;
533c21
     int task_id = 0;
533c21
     int target_rc = 0;
533c21
+    int old_target_rc = 0;
533c21
     int status = PCMK_EXEC_UNKNOWN;
533c21
     guint interval_ms = 0;
533c21
     const char *task = NULL;
533c21
@@ -3671,6 +3678,7 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
533c21
     bool expired = false;
533c21
     pe_resource_t *parent = rsc;
533c21
     enum action_fail_response failure_strategy = action_fail_recover;
533c21
+    bool maskable_probe_failure = false;
533c21
 
533c21
     CRM_CHECK(rsc && node && xml_op, return);
533c21
 
533c21
@@ -3727,10 +3735,22 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
533c21
         expired = true;
533c21
     }
533c21
 
533c21
+    old_rc = rc;
533c21
+    old_target_rc = target_rc;
533c21
+
533c21
     remap_operation(xml_op, rsc, node, data_set, on_fail, target_rc,
533c21
                     &rc, &status);
533c21
 
533c21
-    if (expired && (rc != target_rc)) {
533c21
+    maskable_probe_failure = !pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op);
533c21
+
533c21
+    if (expired && maskable_probe_failure && old_rc != old_target_rc) {
533c21
+        if (rsc->role <= RSC_ROLE_STOPPED) {
533c21
+            rsc->role = RSC_ROLE_UNKNOWN;
533c21
+        }
533c21
+
533c21
+        goto done;
533c21
+
533c21
+    } else if (expired && (rc != target_rc)) {
533c21
         const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
533c21
 
533c21
         if (interval_ms == 0) {
533c21
@@ -3758,6 +3778,18 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
533c21
         }
533c21
     }
533c21
 
533c21
+    if (maskable_probe_failure) {
533c21
+        crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
533c21
+                   services_ocf_exitcode_str(rc), rsc->id, node->details->uname);
533c21
+        update_resource_state(rsc, node, xml_op, task, target_rc, *last_failure,
533c21
+                              on_fail, data_set);
533c21
+        crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
533c21
+
533c21
+        record_failed_op(xml_op, node, rsc, data_set);
533c21
+        resource_location(parent, node, -INFINITY, "masked-probe-failure", data_set);
533c21
+        goto done;
533c21
+    }
533c21
+
533c21
     switch (status) {
533c21
         case PCMK_EXEC_CANCELLED:
533c21
             // Should never happen
533c21
-- 
533c21
2.27.0
533c21
533c21
533c21
From ccff6eb60598f389008b0621447056457da79671 Mon Sep 17 00:00:00 2001
533c21
From: Chris Lumens <clumens@redhat.com>
533c21
Date: Tue, 4 Jan 2022 10:14:48 -0500
533c21
Subject: [PATCH 21/21] Test: scheduler: Add tests for expired, masked probe
533c21
 failures.
533c21
533c21
dummy-1 is a stopped resource with an expired masked probe failure.
533c21
This probe should be rescheduled.  dummy-2 is a started resource with an
533c21
expired masked probe failure.  This probe should not be rescheduled.
533c21
---
533c21
 cts/cts-scheduler.in                          |  1 +
533c21
 .../dot/expired-failed-probe-primitive.dot    |  8 ++
533c21
 .../exp/expired-failed-probe-primitive.exp    | 45 ++++++++++++
533c21
 .../expired-failed-probe-primitive.scores     |  7 ++
533c21
 .../expired-failed-probe-primitive.summary    | 26 +++++++
533c21
 .../xml/expired-failed-probe-primitive.xml    | 73 +++++++++++++++++++
533c21
 6 files changed, 160 insertions(+)
533c21
 create mode 100644 cts/scheduler/dot/expired-failed-probe-primitive.dot
533c21
 create mode 100644 cts/scheduler/exp/expired-failed-probe-primitive.exp
533c21
 create mode 100644 cts/scheduler/scores/expired-failed-probe-primitive.scores
533c21
 create mode 100644 cts/scheduler/summary/expired-failed-probe-primitive.summary
533c21
 create mode 100644 cts/scheduler/xml/expired-failed-probe-primitive.xml
533c21
533c21
diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
533c21
index 3abcbc6c9d..7bc41a0936 100644
533c21
--- a/cts/cts-scheduler.in
533c21
+++ b/cts/cts-scheduler.in
533c21
@@ -115,6 +115,7 @@ TESTS = [
533c21
         [ "probe-pending-node", "Probe (pending node + unmanaged resource)" ],
533c21
         [ "failed-probe-primitive", "Maskable vs. unmaskable probe failures on primitive resources" ],
533c21
         [ "failed-probe-clone", "Maskable vs. unmaskable probe failures on cloned resources" ],
533c21
+        [ "expired-failed-probe-primitive", "Maskable, expired probe failure on primitive resources" ],
533c21
         [ "standby", "Standby" ],
533c21
         [ "comments", "Comments" ],
533c21
     ],
533c21
diff --git a/cts/scheduler/dot/expired-failed-probe-primitive.dot b/cts/scheduler/dot/expired-failed-probe-primitive.dot
533c21
new file mode 100644
533c21
index 0000000000..610c2b8047
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/dot/expired-failed-probe-primitive.dot
533c21
@@ -0,0 +1,8 @@
533c21
+ digraph "g" {
533c21
+"dummy-1_monitor_0 cluster01" -> "dummy-1_start_0 cluster02" [ style = bold]
533c21
+"dummy-1_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"]
533c21
+"dummy-1_monitor_0 cluster02" -> "dummy-1_start_0 cluster02" [ style = bold]
533c21
+"dummy-1_monitor_0 cluster02" [ style=bold color="green" fontcolor="black"]
533c21
+"dummy-1_start_0 cluster02" [ style=bold color="green" fontcolor="black"]
533c21
+"dummy-2_monitor_0 cluster01" [ style=bold color="green" fontcolor="black"]
533c21
+}
533c21
diff --git a/cts/scheduler/exp/expired-failed-probe-primitive.exp b/cts/scheduler/exp/expired-failed-probe-primitive.exp
533c21
new file mode 100644
533c21
index 0000000000..3c2cbfe411
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/exp/expired-failed-probe-primitive.exp
533c21
@@ -0,0 +1,45 @@
533c21
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
533c21
+  <synapse id="0">
533c21
+    <action_set>
533c21
+      <rsc_op id="7" operation="start" operation_key="dummy-1_start_0" on_node="cluster02" on_node_uuid="2">
533c21
+        <primitive id="dummy-1" class="ocf" provider="pacemaker" type="Dummy"/>
533c21
+        <attributes CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" />
533c21
+      </rsc_op>
533c21
+    </action_set>
533c21
+    <inputs>
533c21
+      <trigger>
533c21
+        <rsc_op id="2" operation="monitor" operation_key="dummy-1_monitor_0" on_node="cluster01" on_node_uuid="1"/>
533c21
+      </trigger>
533c21
+      <trigger>
533c21
+        <rsc_op id="4" operation="monitor" operation_key="dummy-1_monitor_0" on_node="cluster02" on_node_uuid="2"/>
533c21
+      </trigger>
533c21
+    </inputs>
533c21
+  </synapse>
533c21
+  <synapse id="1">
533c21
+    <action_set>
533c21
+      <rsc_op id="4" operation="monitor" operation_key="dummy-1_monitor_0" on_node="cluster02" on_node_uuid="2">
533c21
+        <primitive id="dummy-1" class="ocf" provider="pacemaker" type="Dummy"/>
533c21
+        <attributes CRM_meta_on_node="cluster02" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
533c21
+      </rsc_op>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+  <synapse id="2">
533c21
+    <action_set>
533c21
+      <rsc_op id="2" operation="monitor" operation_key="dummy-1_monitor_0" on_node="cluster01" on_node_uuid="1">
533c21
+        <primitive id="dummy-1" class="ocf" provider="pacemaker" type="Dummy"/>
533c21
+        <attributes CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
533c21
+      </rsc_op>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+  <synapse id="3">
533c21
+    <action_set>
533c21
+      <rsc_op id="3" operation="monitor" operation_key="dummy-2_monitor_0" on_node="cluster01" on_node_uuid="1">
533c21
+        <primitive id="dummy-2" class="ocf" provider="pacemaker" type="Dummy"/>
533c21
+        <attributes CRM_meta_on_node="cluster01" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
533c21
+      </rsc_op>
533c21
+    </action_set>
533c21
+    <inputs/>
533c21
+  </synapse>
533c21
+</transition_graph>
533c21
diff --git a/cts/scheduler/scores/expired-failed-probe-primitive.scores b/cts/scheduler/scores/expired-failed-probe-primitive.scores
533c21
new file mode 100644
533c21
index 0000000000..51ae5510e6
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/scores/expired-failed-probe-primitive.scores
533c21
@@ -0,0 +1,7 @@
533c21
+
533c21
+pcmk__native_allocate: Fencing allocation score on cluster01: 0
533c21
+pcmk__native_allocate: Fencing allocation score on cluster02: 0
533c21
+pcmk__native_allocate: dummy-1 allocation score on cluster01: 0
533c21
+pcmk__native_allocate: dummy-1 allocation score on cluster02: 0
533c21
+pcmk__native_allocate: dummy-2 allocation score on cluster01: 0
533c21
+pcmk__native_allocate: dummy-2 allocation score on cluster02: 0
533c21
diff --git a/cts/scheduler/summary/expired-failed-probe-primitive.summary b/cts/scheduler/summary/expired-failed-probe-primitive.summary
533c21
new file mode 100644
533c21
index 0000000000..ac0604e84f
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/summary/expired-failed-probe-primitive.summary
533c21
@@ -0,0 +1,26 @@
533c21
+Current cluster status:
533c21
+  * Node List:
533c21
+    * Online: [ cluster01 cluster02 ]
533c21
+
533c21
+  * Full List of Resources:
533c21
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
+    * dummy-1	(ocf:pacemaker:Dummy):	 Stopped
533c21
+    * dummy-2	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
+
533c21
+Transition Summary:
533c21
+  * Start      dummy-1     ( cluster02 )
533c21
+
533c21
+Executing Cluster Transition:
533c21
+  * Resource action: dummy-1         monitor on cluster02
533c21
+  * Resource action: dummy-1         monitor on cluster01
533c21
+  * Resource action: dummy-2         monitor on cluster01
533c21
+  * Resource action: dummy-1         start on cluster02
533c21
+
533c21
+Revised Cluster Status:
533c21
+  * Node List:
533c21
+    * Online: [ cluster01 cluster02 ]
533c21
+
533c21
+  * Full List of Resources:
533c21
+    * Fencing	(stonith:fence_xvm):	 Started cluster01
533c21
+    * dummy-1	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
+    * dummy-2	(ocf:pacemaker:Dummy):	 Started cluster02
533c21
diff --git a/cts/scheduler/xml/expired-failed-probe-primitive.xml b/cts/scheduler/xml/expired-failed-probe-primitive.xml
533c21
new file mode 100644
533c21
index 0000000000..684aa73f92
533c21
--- /dev/null
533c21
+++ b/cts/scheduler/xml/expired-failed-probe-primitive.xml
533c21
@@ -0,0 +1,73 @@
533c21
+<cib crm_feature_set="3.3.0" validate-with="pacemaker-3.3" epoch="1" num_updates="37" admin_epoch="1" cib-last-written="Tue Jan  4 10:00:00 2021" update-origin="cluster01" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2">
533c21
+  <configuration>
533c21
+    <crm_config>
533c21
+      <cluster_property_set id="cib-bootstrap-options">
533c21
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
533c21
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-1.e97f9675f.git.el7-e97f9675f"/>
533c21
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
533c21
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="test-cluster"/>
533c21
+        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
533c21
+        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
533c21
+      </cluster_property_set>
533c21
+    </crm_config>
533c21
+    <nodes>
533c21
+      <node id="1" uname="cluster01"/>
533c21
+      <node id="2" uname="cluster02"/>
533c21
+    </nodes>
533c21
+    <resources>
533c21
+      <primitive class="stonith" id="Fencing" type="fence_xvm">
533c21
+        <instance_attributes id="Fencing-instance_attributes">
533c21
+          <nvpair id="Fencing-instance_attributes-ip_family" name="ip_family" value="ipv4"/>
533c21
+        </instance_attributes>
533c21
+        <operations>
533c21
+          <op id="Fencing-monitor-interval-60s" interval="60s" name="monitor"/>
533c21
+        </operations>
533c21
+      </primitive>
533c21
+      <primitive class="ocf" id="dummy-1" provider="pacemaker" type="Dummy">
533c21
+        <meta_attributes id="dummy-1-meta_attributes">
533c21
+          <nvpair id="dummy-1-meta_attributes-failure-timeout" name="failure-timeout" value="10"/>
533c21
+        </meta_attributes>
533c21
+      </primitive>
533c21
+      <primitive class="ocf" id="dummy-2" provider="pacemaker" type="Dummy">
533c21
+        <meta_attributes id="dummy-2-meta_attributes">
533c21
+          <nvpair id="dummy-2-meta_attributes-failure-timeout" name="failure-timeout" value="10"/>
533c21
+        </meta_attributes>
533c21
+      </primitive>
533c21
+    </resources>
533c21
+    <constraints/>
533c21
+  </configuration>
533c21
+  <status>
533c21
+    <node_state id="1" uname="cluster01" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
533c21
+      <lrm id="1">
533c21
+        <lrm_resources>
533c21
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
533c21
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="3" rc-code="0" op-status="0" interval="0" last-rc-change="1588951272" exec-time="36" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
533c21
+            <lrm_rsc_op id="Fencing_monitor_60000" operation_key="Fencing_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="4" rc-code="0" op-status="0" interval="60000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="a88218bb6c7dc47e6586fc75fc2a8d69"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-1_last_failure_0" operation_key="dummy-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="5" rc-code="5" op-status="0" interval="0" last-rc-change="1590608589" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;5:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="6" rc-code="5" op-status="0" interval="0" last-rc-change="1590608589" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+        </lrm_resources>
533c21
+      </lrm>
533c21
+    </node_state>
533c21
+    <node_state id="2" uname="cluster02" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
533c21
+      <lrm id="2">
533c21
+        <lrm_resources>
533c21
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
533c21
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:7;1:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1588951263" exec-time="3" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="dummy-1" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-1_last_failure_0" operation_key="dummy-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="2" rc-code="5" op-status="0" interval="0" last-rc-change="1590608589" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+          <lrm_resource id="dummy-2" class="ocf" provider="pacemaker" type="Dummy">
533c21
+            <lrm_rsc_op id="dummy-2_last_failure_0" operation_key="dummy-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:5;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="7" rc-code="5" op-status="0" interval="0" last-rc-change="1590608589" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+            <lrm_rsc_op id="dummy-2_last_0" operation_key="dummy-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster02" call-id="8" rc-code="0" op-status="0" interval="0" last-rc-change="1590609000" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
533c21
+          </lrm_resource>
533c21
+        </lrm_resources>
533c21
+      </lrm>
533c21
+    </node_state>
533c21
+  </status>
533c21
+</cib>
533c21
-- 
533c21
2.27.0
533c21