From f1f71b3f3c342987db0058e7db0030417f3f83fa Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Thu, 28 May 2020 08:22:00 -0500
Subject: [PATCH 01/20] Refactor: scheduler: functionize comparing on-fail
values
The action_fail_response enum values used for the "on-fail" operation
meta-attribute were initially intended to be in order of severity.
However as new values were added, they were added to the end (out of severity
order) to preserve API backward compatibility.
This resulted in a convoluted comparison of values that will only get worse as
more values are added.
This commit adds a comparison function to isolate that complexity.
---
include/crm/pengine/common.h | 32 ++++++++++++------
lib/pengine/unpack.c | 80 +++++++++++++++++++++++++++++++++++++++++---
2 files changed, 97 insertions(+), 15 deletions(-)
diff --git a/include/crm/pengine/common.h b/include/crm/pengine/common.h
index 3a770b7..2737b2e 100644
--- a/include/crm/pengine/common.h
+++ b/include/crm/pengine/common.h
@@ -22,18 +22,29 @@ extern "C" {
extern gboolean was_processing_error;
extern gboolean was_processing_warning;
-/* order is significant here
- * items listed in order of accending severeness
- * more severe actions take precedent over lower ones
+/* The order is (partially) significant here; the values from action_fail_ignore
+ * through action_fail_fence are in order of increasing severity.
+ *
+ * @COMPAT The values should be ordered and numbered per the "TODO" comments
+ * below, so all values are in order of severity and there is room for
+ * future additions, but that would break API compatibility.
+ * @TODO For now, we just use a function to compare the values specially, but
+ * at the next compatibility break, we should arrange things properly.
*/
enum action_fail_response {
- action_fail_ignore,
- action_fail_recover,
- action_fail_migrate, /* recover by moving it somewhere else */
- action_fail_block,
- action_fail_stop,
- action_fail_standby,
- action_fail_fence,
+ action_fail_ignore, // @TODO = 10
+ // @TODO action_fail_demote = 20,
+ action_fail_recover, // @TODO = 30
+ // @TODO action_fail_reset_remote = 40,
+ // @TODO action_fail_restart_container = 50,
+ action_fail_migrate, // @TODO = 60
+ action_fail_block, // @TODO = 70
+ action_fail_stop, // @TODO = 80
+ action_fail_standby, // @TODO = 90
+ action_fail_fence, // @TODO = 100
+
+ // @COMPAT Values below here are out of order for API compatibility
+
action_fail_restart_container,
/* This is reserved for internal use for remote node connection resources.
@@ -44,6 +55,7 @@ enum action_fail_response {
*/
action_fail_reset_remote,
+ action_fail_demote,
};
/* the "done" action must be the "pre" action +1 */
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index 3c6606b..f688881 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -2770,6 +2770,78 @@ last_change_str(xmlNode *xml_op)
return ((when_s && *when_s)? when_s : "unknown time");
}
+/*!
+ * \internal
+ * \brief Compare two on-fail values
+ *
+ * \param[in] first One on-fail value to compare
+ * \param[in] second The other on-fail value to compare
+ *
+ * \return A negative number if second is more severe than first, zero if they
+ * are equal, or a positive number if first is more severe than second.
+ * \note This is only needed until the action_fail_response values can be
+ * renumbered at the next API compatibility break.
+ */
+static int
+cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
+{
+ switch (first) {
+ case action_fail_reset_remote:
+ switch (second) {
+ case action_fail_ignore:
+ case action_fail_recover:
+ return 1;
+ case action_fail_reset_remote:
+ return 0;
+ default:
+ return -1;
+ }
+ break;
+
+ case action_fail_restart_container:
+ switch (second) {
+ case action_fail_ignore:
+ case action_fail_recover:
+ case action_fail_reset_remote:
+ return 1;
+ case action_fail_restart_container:
+ return 0;
+ default:
+ return -1;
+ }
+ break;
+
+ default:
+ break;
+ }
+ switch (second) {
+ case action_fail_reset_remote:
+ switch (first) {
+ case action_fail_ignore:
+ case action_fail_recover:
+ return -1;
+ default:
+ return 1;
+ }
+ break;
+
+ case action_fail_restart_container:
+ switch (first) {
+ case action_fail_ignore:
+ case action_fail_recover:
+ case action_fail_reset_remote:
+ return -1;
+ default:
+ return 1;
+ }
+ break;
+
+ default:
+ break;
+ }
+ return first - second;
+}
+
static void
unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * xml_op, xmlNode ** last_failure,
enum action_fail_response * on_fail, pe_working_set_t * data_set)
@@ -2829,10 +2901,7 @@ unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * x
}
action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
- if ((action->on_fail <= action_fail_fence && *on_fail < action->on_fail) ||
- (action->on_fail == action_fail_reset_remote && *on_fail <= action_fail_recover) ||
- (action->on_fail == action_fail_restart_container && *on_fail <= action_fail_recover) ||
- (*on_fail == action_fail_restart_container && action->on_fail >= action_fail_migrate)) {
+ if (cmp_on_fail(*on_fail, action->on_fail) < 0) {
pe_rsc_trace(rsc, "on-fail %s -> %s for %s (%s)", fail2text(*on_fail),
fail2text(action->on_fail), action->uuid, key);
*on_fail = action->on_fail;
@@ -3675,7 +3744,8 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
record_failed_op(xml_op, node, rsc, data_set);
- if (failure_strategy == action_fail_restart_container && *on_fail <= action_fail_recover) {
+ if ((failure_strategy == action_fail_restart_container)
+ && cmp_on_fail(*on_fail, action_fail_recover) <= 0) {
*on_fail = failure_strategy;
}
--
1.8.3.1
From ef246ff05d7459f9672b10ac1873e3191a3b46e9 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Thu, 28 May 2020 08:27:47 -0500
Subject: [PATCH 02/20] Fix: scheduler: disallow on-fail=stop for stop
operations
because it would loop infinitely as long as the stop continued to fail
---
lib/pengine/utils.c | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index 20a8db5..3fb7e62 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -716,16 +716,25 @@ custom_action(pe_resource_t * rsc, char *key, const char *task,
return action;
}
+static bool
+valid_stop_on_fail(const char *value)
+{
+ return safe_str_neq(value, "standby")
+ && safe_str_neq(value, "stop");
+}
+
static const char *
unpack_operation_on_fail(pe_action_t * action)
{
const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);
- if (safe_str_eq(action->task, CRMD_ACTION_STOP) && safe_str_eq(value, "standby")) {
+ if (safe_str_eq(action->task, CRMD_ACTION_STOP)
+ && !valid_stop_on_fail(value)) {
+
pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop "
- "action to default value because 'standby' is not "
- "allowed for stop", action->rsc->id);
+ "action to default value because '%s' is not "
+ "allowed for stop", action->rsc->id, value);
return NULL;
} else if (safe_str_eq(action->task, CRMD_ACTION_DEMOTE) && !value) {
/* demote on_fail defaults to master monitor value if present */
--
1.8.3.1
From 8dceba792ffe65cd77c3aae430067638dbba63f9 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Thu, 28 May 2020 08:50:33 -0500
Subject: [PATCH 03/20] Refactor: scheduler: use more appropriate types in a
couple places
---
lib/pengine/unpack.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index f688881..6a350e5 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -2244,7 +2244,7 @@ unpack_lrm_rsc_state(pe_node_t * node, xmlNode * rsc_entry, pe_working_set_t * d
xmlNode *rsc_op = NULL;
xmlNode *last_failure = NULL;
- enum action_fail_response on_fail = FALSE;
+ enum action_fail_response on_fail = action_fail_ignore;
enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;
crm_trace("[%s] Processing %s on %s",
@@ -2287,7 +2287,6 @@ unpack_lrm_rsc_state(pe_node_t * node, xmlNode * rsc_entry, pe_working_set_t * d
/* process operations */
saved_role = rsc->role;
- on_fail = action_fail_ignore;
rsc->role = RSC_ROLE_UNKNOWN;
sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
@@ -3376,7 +3375,7 @@ int pe__target_rc_from_xml(xmlNode *xml_op)
static enum action_fail_response
get_action_on_fail(pe_resource_t *rsc, const char *key, const char *task, pe_working_set_t * data_set)
{
- int result = action_fail_recover;
+ enum action_fail_response result = action_fail_recover;
pe_action_t *action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
result = action->on_fail;
--
1.8.3.1
From a4d6a20a990d1461184f888e21aa61cddff8996d Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 2 Jun 2020 12:05:57 -0500
Subject: [PATCH 04/20] Low: libpacemaker: don't force stop when skipping
reload of failed resource
Normal failure recovery will apply, which will stop if needed.
(The stop was forced as of 2558d76f.)
---
lib/pacemaker/pcmk_sched_native.c | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c
index bd8a0b5..ff2fb92 100644
--- a/lib/pacemaker/pcmk_sched_native.c
+++ b/lib/pacemaker/pcmk_sched_native.c
@@ -3362,9 +3362,19 @@ ReloadRsc(pe_resource_t * rsc, pe_node_t *node, pe_working_set_t * data_set)
pe_rsc_trace(rsc, "%s: unmanaged", rsc->id);
return;
- } else if (is_set(rsc->flags, pe_rsc_failed) || is_set(rsc->flags, pe_rsc_start_pending)) {
- pe_rsc_trace(rsc, "%s: general resource state: flags=0x%.16llx", rsc->id, rsc->flags);
- stop_action(rsc, node, FALSE); /* Force a full restart, overkill? */
+ } else if (is_set(rsc->flags, pe_rsc_failed)) {
+ /* We don't need to specify any particular actions here, normal failure
+ * recovery will apply.
+ */
+ pe_rsc_trace(rsc, "%s: preventing reload because failed", rsc->id);
+ return;
+
+ } else if (is_set(rsc->flags, pe_rsc_start_pending)) {
+ /* If a resource's configuration changed while a start was pending,
+ * force a full restart.
+ */
+ pe_rsc_trace(rsc, "%s: preventing reload because start pending", rsc->id);
+ stop_action(rsc, node, FALSE);
return;
} else if (node == NULL) {
--
1.8.3.1
From f2d244bc4306297d5960c0ba54e0a85a68e864ee Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 2 Jun 2020 12:16:33 -0500
Subject: [PATCH 05/20] Test: scheduler: test forcing a restart instead of
reload when start is pending
---
cts/cts-scheduler.in | 1 +
cts/scheduler/params-3.dot | 28 ++++++
cts/scheduler/params-3.exp | 208 +++++++++++++++++++++++++++++++++++++++++
cts/scheduler/params-3.scores | 21 +++++
cts/scheduler/params-3.summary | 45 +++++++++
cts/scheduler/params-3.xml | 154 ++++++++++++++++++++++++++++++
6 files changed, 457 insertions(+)
create mode 100644 cts/scheduler/params-3.dot
create mode 100644 cts/scheduler/params-3.exp
create mode 100644 cts/scheduler/params-3.scores
create mode 100644 cts/scheduler/params-3.summary
create mode 100644 cts/scheduler/params-3.xml
diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
index 346ada2..ae8247e 100644
--- a/cts/cts-scheduler.in
+++ b/cts/cts-scheduler.in
@@ -84,6 +84,7 @@ TESTS = [
[ "params-0", "Params: No change" ],
[ "params-1", "Params: Changed" ],
[ "params-2", "Params: Resource definition" ],
+ [ "params-3", "Params: Restart instead of reload if start pending" ],
[ "params-4", "Params: Reload" ],
[ "params-5", "Params: Restart based on probe digest" ],
[ "novell-251689", "Resource definition change + target_role=stopped" ],
diff --git a/cts/scheduler/params-3.dot b/cts/scheduler/params-3.dot
new file mode 100644
index 0000000..d681ee5
--- /dev/null
+++ b/cts/scheduler/params-3.dot
@@ -0,0 +1,28 @@
+ digraph "g" {
+"Cancel rsc_c001n02_monitor_5000 c001n02" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_monitor_0 c001n01" -> "DcIPaddr_start_0 c001n02" [ style = bold]
+"DcIPaddr_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_monitor_0 c001n03" -> "DcIPaddr_start_0 c001n02" [ style = bold]
+"DcIPaddr_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_monitor_0 c001n08" -> "DcIPaddr_start_0 c001n02" [ style = bold]
+"DcIPaddr_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_monitor_5000 c001n02" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_start_0 c001n02" -> "DcIPaddr_monitor_5000 c001n02" [ style = bold]
+"DcIPaddr_start_0 c001n02" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_stop_0 c001n02" -> "DcIPaddr_start_0 c001n02" [ style = bold]
+"DcIPaddr_stop_0 c001n02" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n01_monitor_0 c001n02" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n01_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n01_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n02_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n02_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n02_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n02_monitor_6000 c001n02" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n03_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n03_monitor_0 c001n02" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n03_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n08_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n08_monitor_0 c001n02" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n08_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n08_monitor_5000 c001n08" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/cts/scheduler/params-3.exp b/cts/scheduler/params-3.exp
new file mode 100644
index 0000000..5cccdec
--- /dev/null
+++ b/cts/scheduler/params-3.exp
@@ -0,0 +1,208 @@
+<transition_graph cluster-delay="3m" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="22" operation="monitor" operation_key="DcIPaddr_monitor_5000" on_node="c001n02" on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f">
+ <primitive id="DcIPaddr" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_interval="5000" CRM_meta_name="monitor" CRM_meta_on_node="c001n02" CRM_meta_on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f" CRM_meta_timeout="20000" ip="127.0.0.20"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="21" operation="start" operation_key="DcIPaddr_start_0" on_node="c001n02" on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="21" operation="start" operation_key="DcIPaddr_start_0" on_node="c001n02" on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f">
+ <primitive id="DcIPaddr" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n02" CRM_meta_on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f" CRM_meta_timeout="20000" ip="127.0.0.20"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="5" operation="stop" operation_key="DcIPaddr_stop_0" on_node="c001n02" on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="6" operation="monitor" operation_key="DcIPaddr_monitor_0" on_node="c001n01" on_node_uuid="de937e3d-0309-4b5d-b85c-f96edc1ed8e3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="13" operation="monitor" operation_key="DcIPaddr_monitor_0" on_node="c001n03" on_node_uuid="5d9a8c11-8684-43ea-91.0.6e221530c193"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="17" operation="monitor" operation_key="DcIPaddr_monitor_0" on_node="c001n08" on_node_uuid="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="17" operation="monitor" operation_key="DcIPaddr_monitor_0" on_node="c001n08" on_node_uuid="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b">
+ <primitive id="DcIPaddr" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n08" CRM_meta_on_node_uuid="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.20"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="13" operation="monitor" operation_key="DcIPaddr_monitor_0" on_node="c001n03" on_node_uuid="5d9a8c11-8684-43ea-91.0.6e221530c193">
+ <primitive id="DcIPaddr" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n03" CRM_meta_on_node_uuid="5d9a8c11-8684-43ea-91.0.6e221530c193" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.20"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <rsc_op id="6" operation="monitor" operation_key="DcIPaddr_monitor_0" on_node="c001n01" on_node_uuid="de937e3d-0309-4b5d-b85c-f96edc1ed8e3">
+ <primitive id="DcIPaddr" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n01" CRM_meta_on_node_uuid="de937e3d-0309-4b5d-b85c-f96edc1ed8e3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.20"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <rsc_op id="5" operation="stop" operation_key="DcIPaddr_stop_0" on_node="c001n02" on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f">
+ <primitive id="DcIPaddr" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n02" CRM_meta_on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f" CRM_meta_timeout="20000" ip="127.0.0.20"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <rsc_op id="14" operation="monitor" operation_key="rsc_c001n08_monitor_0" on_node="c001n03" on_node_uuid="5d9a8c11-8684-43ea-91.0.6e221530c193">
+ <primitive id="rsc_c001n08" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n03" CRM_meta_on_node_uuid="5d9a8c11-8684-43ea-91.0.6e221530c193" CRM_meta_op_target_rc="7" CRM_meta_timeout="21000" ip="127.0.0.11"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <rsc_op id="10" operation="monitor" operation_key="rsc_c001n08_monitor_0" on_node="c001n02" on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f">
+ <primitive id="rsc_c001n08" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n02" CRM_meta_on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f" CRM_meta_op_target_rc="7" CRM_meta_timeout="21000" ip="127.0.0.11"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <rsc_op id="7" operation="monitor" operation_key="rsc_c001n08_monitor_0" on_node="c001n01" on_node_uuid="de937e3d-0309-4b5d-b85c-f96edc1ed8e3">
+ <primitive id="rsc_c001n08" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n01" CRM_meta_on_node_uuid="de937e3d-0309-4b5d-b85c-f96edc1ed8e3" CRM_meta_op_target_rc="7" CRM_meta_timeout="21000" ip="127.0.0.11"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <rsc_op id="1" operation="monitor" operation_key="rsc_c001n08_monitor_5000" on_node="c001n08" on_node_uuid="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b">
+ <primitive id="rsc_c001n08" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_interval="5000" CRM_meta_name="monitor" CRM_meta_on_node="c001n08" CRM_meta_on_node_uuid="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b" CRM_meta_timeout="21000" ip="127.0.0.11" new_attr="some_value"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="10">
+ <action_set>
+ <rsc_op id="27" operation="monitor" operation_key="rsc_c001n02_monitor_6000" on_node="c001n02" on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f">
+ <primitive id="rsc_c001n02" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_interval="6000" CRM_meta_name="monitor" CRM_meta_on_node="c001n02" CRM_meta_on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f" CRM_meta_timeout="20000" ip="127.0.0.12"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="11">
+ <action_set>
+ <rsc_op id="18" operation="monitor" operation_key="rsc_c001n02_monitor_0" on_node="c001n08" on_node_uuid="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b">
+ <primitive id="rsc_c001n02" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n08" CRM_meta_on_node_uuid="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.12"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="12">
+ <action_set>
+ <rsc_op id="15" operation="monitor" operation_key="rsc_c001n02_monitor_0" on_node="c001n03" on_node_uuid="5d9a8c11-8684-43ea-91.0.6e221530c193">
+ <primitive id="rsc_c001n02" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n03" CRM_meta_on_node_uuid="5d9a8c11-8684-43ea-91.0.6e221530c193" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.12"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="13">
+ <action_set>
+ <rsc_op id="8" operation="monitor" operation_key="rsc_c001n02_monitor_0" on_node="c001n01" on_node_uuid="de937e3d-0309-4b5d-b85c-f96edc1ed8e3">
+ <primitive id="rsc_c001n02" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n01" CRM_meta_on_node_uuid="de937e3d-0309-4b5d-b85c-f96edc1ed8e3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.12"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="14">
+ <action_set>
+ <rsc_op id="2" operation="cancel" operation_key="rsc_c001n02_monitor_5000" on_node="c001n02" on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f">
+ <primitive id="rsc_c001n02" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_call_id="6" CRM_meta_interval="5000" CRM_meta_on_node="c001n02" CRM_meta_on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f" CRM_meta_operation="monitor" CRM_meta_timeout="20000" ip="127.0.0.12"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="15">
+ <action_set>
+ <rsc_op id="19" operation="monitor" operation_key="rsc_c001n03_monitor_0" on_node="c001n08" on_node_uuid="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b">
+ <primitive id="rsc_c001n03" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n08" CRM_meta_on_node_uuid="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.13"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="16">
+ <action_set>
+ <rsc_op id="11" operation="monitor" operation_key="rsc_c001n03_monitor_0" on_node="c001n02" on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f">
+ <primitive id="rsc_c001n03" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n02" CRM_meta_on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.13"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="17">
+ <action_set>
+ <rsc_op id="9" operation="monitor" operation_key="rsc_c001n03_monitor_0" on_node="c001n01" on_node_uuid="de937e3d-0309-4b5d-b85c-f96edc1ed8e3">
+ <primitive id="rsc_c001n03" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n01" CRM_meta_on_node_uuid="de937e3d-0309-4b5d-b85c-f96edc1ed8e3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.13"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="18">
+ <action_set>
+ <rsc_op id="20" operation="monitor" operation_key="rsc_c001n01_monitor_0" on_node="c001n08" on_node_uuid="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b">
+ <primitive id="rsc_c001n01" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n08" CRM_meta_on_node_uuid="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.14"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="19">
+ <action_set>
+ <rsc_op id="16" operation="monitor" operation_key="rsc_c001n01_monitor_0" on_node="c001n03" on_node_uuid="5d9a8c11-8684-43ea-91.0.6e221530c193">
+ <primitive id="rsc_c001n01" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n03" CRM_meta_on_node_uuid="5d9a8c11-8684-43ea-91.0.6e221530c193" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.14"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="20">
+ <action_set>
+ <rsc_op id="12" operation="monitor" operation_key="rsc_c001n01_monitor_0" on_node="c001n02" on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f">
+ <primitive id="rsc_c001n01" class="ocf" provider="heartbeat" type="IPaddr"/>
+ <attributes CRM_meta_on_node="c001n02" CRM_meta_on_node_uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" ip="127.0.0.14"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+</transition_graph>
diff --git a/cts/scheduler/params-3.scores b/cts/scheduler/params-3.scores
new file mode 100644
index 0000000..00417ea
--- /dev/null
+++ b/cts/scheduler/params-3.scores
@@ -0,0 +1,21 @@
+Allocation scores:
+pcmk__native_allocate: DcIPaddr allocation score on c001n01: -INFINITY
+pcmk__native_allocate: DcIPaddr allocation score on c001n02: 0
+pcmk__native_allocate: DcIPaddr allocation score on c001n03: -INFINITY
+pcmk__native_allocate: DcIPaddr allocation score on c001n08: -INFINITY
+pcmk__native_allocate: rsc_c001n01 allocation score on c001n01: 100
+pcmk__native_allocate: rsc_c001n01 allocation score on c001n02: 0
+pcmk__native_allocate: rsc_c001n01 allocation score on c001n03: 0
+pcmk__native_allocate: rsc_c001n01 allocation score on c001n08: 0
+pcmk__native_allocate: rsc_c001n02 allocation score on c001n01: 0
+pcmk__native_allocate: rsc_c001n02 allocation score on c001n02: 100
+pcmk__native_allocate: rsc_c001n02 allocation score on c001n03: 0
+pcmk__native_allocate: rsc_c001n02 allocation score on c001n08: 0
+pcmk__native_allocate: rsc_c001n03 allocation score on c001n01: 0
+pcmk__native_allocate: rsc_c001n03 allocation score on c001n02: 0
+pcmk__native_allocate: rsc_c001n03 allocation score on c001n03: 100
+pcmk__native_allocate: rsc_c001n03 allocation score on c001n08: 0
+pcmk__native_allocate: rsc_c001n08 allocation score on c001n01: 0
+pcmk__native_allocate: rsc_c001n08 allocation score on c001n02: 0
+pcmk__native_allocate: rsc_c001n08 allocation score on c001n03: 0
+pcmk__native_allocate: rsc_c001n08 allocation score on c001n08: 100
diff --git a/cts/scheduler/params-3.summary b/cts/scheduler/params-3.summary
new file mode 100644
index 0000000..257f8ba
--- /dev/null
+++ b/cts/scheduler/params-3.summary
@@ -0,0 +1,45 @@
+
+Current cluster status:
+Online: [ c001n01 c001n02 c001n03 c001n08 ]
+
+ DcIPaddr (ocf::heartbeat:IPaddr): Starting c001n02
+ rsc_c001n08 (ocf::heartbeat:IPaddr): Started c001n08
+ rsc_c001n02 (ocf::heartbeat:IPaddr): Started c001n02
+ rsc_c001n03 (ocf::heartbeat:IPaddr): Started c001n03
+ rsc_c001n01 (ocf::heartbeat:IPaddr): Started c001n01
+
+Transition Summary:
+ * Restart DcIPaddr ( c001n02 )
+
+Executing cluster transition:
+ * Resource action: DcIPaddr monitor on c001n08
+ * Resource action: DcIPaddr monitor on c001n03
+ * Resource action: DcIPaddr monitor on c001n01
+ * Resource action: DcIPaddr stop on c001n02
+ * Resource action: rsc_c001n08 monitor on c001n03
+ * Resource action: rsc_c001n08 monitor on c001n02
+ * Resource action: rsc_c001n08 monitor on c001n01
+ * Resource action: rsc_c001n08 monitor=5000 on c001n08
+ * Resource action: rsc_c001n02 monitor=6000 on c001n02
+ * Resource action: rsc_c001n02 monitor on c001n08
+ * Resource action: rsc_c001n02 monitor on c001n03
+ * Resource action: rsc_c001n02 monitor on c001n01
+ * Resource action: rsc_c001n02 cancel=5000 on c001n02
+ * Resource action: rsc_c001n03 monitor on c001n08
+ * Resource action: rsc_c001n03 monitor on c001n02
+ * Resource action: rsc_c001n03 monitor on c001n01
+ * Resource action: rsc_c001n01 monitor on c001n08
+ * Resource action: rsc_c001n01 monitor on c001n03
+ * Resource action: rsc_c001n01 monitor on c001n02
+ * Resource action: DcIPaddr start on c001n02
+ * Resource action: DcIPaddr monitor=5000 on c001n02
+
+Revised cluster status:
+Online: [ c001n01 c001n02 c001n03 c001n08 ]
+
+ DcIPaddr (ocf::heartbeat:IPaddr): Started c001n02
+ rsc_c001n08 (ocf::heartbeat:IPaddr): Started c001n08
+ rsc_c001n02 (ocf::heartbeat:IPaddr): Started c001n02
+ rsc_c001n03 (ocf::heartbeat:IPaddr): Started c001n03
+ rsc_c001n01 (ocf::heartbeat:IPaddr): Started c001n01
+
diff --git a/cts/scheduler/params-3.xml b/cts/scheduler/params-3.xml
new file mode 100644
index 0000000..ee6e157
--- /dev/null
+++ b/cts/scheduler/params-3.xml
@@ -0,0 +1,154 @@
+<cib admin_epoch="0" epoch="6" num_updates="73" dc-uuid="e9bdfde9-01b0-421f-acd8-8a65a53e775f" have-quorum="true" remote-tls-port="0" validate-with="pacemaker-3.0" cib-last-written="Fri Jul 13 13:51:11 2012">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="no-stonith">
+ <nvpair id="opt-no-stonith" name="stonith-enabled" value="false"/>
+ </cluster_property_set>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="nvpair.id21842" name="cluster-delay" value="3m"/>
+ <nvpair id="nvpair.id21850" name="symetric_cluster" value="true"/>
+ <nvpair id="nvpair.id21860" name="suppress_cib_writes" value="1"/>
+ <nvpair id="nvpair.id21868" name="no-quorum-policy" value="stop"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b" uname="c001n08" type="member"/>
+ <node id="e9bdfde9-01b0-421f-acd8-8a65a53e775f" uname="c001n02" type="member"/>
+ <node id="5d9a8c11-8684-43ea-91.0.6e221530c193" uname="c001n03" type="member"/>
+ <node id="de937e3d-0309-4b5d-b85c-f96edc1ed8e3" uname="c001n01" type="member"/>
+ </nodes>
+ <resources>
+ <primitive id="DcIPaddr" class="ocf" type="IPaddr" provider="heartbeat">
+ <operations>
+ <op name="monitor" interval="5s" id="DcIPaddr-1" timeout="20s"/>
+ </operations>
+ <instance_attributes id="instance_attributes.id21948">
+ <nvpair id="nvpair.id21954" name="ip" value="127.0.0.20"/>
+ </instance_attributes>
+ <meta_attributes id="primitive-DcIPaddr.meta">
+ <nvpair id="is_managed.meta.auto-31" name="is-managed" value="1"/>
+ </meta_attributes>
+ </primitive>
+ <primitive id="rsc_c001n08" class="ocf" type="IPaddr" provider="heartbeat">
+ <operations>
+ <op name="monitor" interval="5s" id="rsc_c001n08-1" timeout="21s">
+ <instance_attributes id="instance_attributes.id21990">
+ <nvpair id="nvpair.id21996" name="new_attr" value="some_value"/>
+ </instance_attributes>
+ </op>
+ </operations>
+ <instance_attributes id="instance_attributes.id22009">
+ <nvpair id="nvpair.id22015" name="ip" value="127.0.0.11"/>
+ </instance_attributes>
+ <meta_attributes id="primitive-rsc_c001n08.meta">
+ <nvpair id="is_managed.meta.auto-47" name="is-managed" value="1"/>
+ </meta_attributes>
+ </primitive>
+ <primitive id="rsc_c001n02" class="ocf" type="IPaddr" provider="heartbeat">
+ <operations>
+ <op name="monitor" interval="6s" id="rsc_c001n02-1" timeout="20s"/>
+ </operations>
+ <instance_attributes id="instance_attributes.id22052">
+ <nvpair id="nvpair.id22059" name="ip" value="127.0.0.12"/>
+ </instance_attributes>
+ <meta_attributes id="primitive-rsc_c001n02.meta">
+ <nvpair id="is_managed.meta.auto-72" name="is-managed" value="1"/>
+ </meta_attributes>
+ </primitive>
+ <primitive id="rsc_c001n03" class="ocf" type="IPaddr" provider="heartbeat">
+ <operations>
+ <op name="monitor" interval="5s" id="rsc_c001n03-1" timeout="20s"/>
+ </operations>
+ <instance_attributes id="instance_attributes.id22096">
+ <nvpair id="nvpair.id22102" name="ip" value="127.0.0.13"/>
+ </instance_attributes>
+ <meta_attributes id="primitive-rsc_c001n03.meta">
+ <nvpair id="is_managed.meta.auto-88" name="is-managed" value="1"/>
+ </meta_attributes>
+ </primitive>
+ <primitive id="rsc_c001n01" class="ocf" type="IPaddr" provider="heartbeat">
+ <operations>
+ <op name="monitor" interval="5s" id="rsc_c001n01-1" timeout="20s"/>
+ </operations>
+ <instance_attributes id="instance_attributes.id22139">
+ <nvpair id="nvpair.id22146" name="ip" value="127.0.0.14"/>
+ </instance_attributes>
+ <meta_attributes id="primitive-rsc_c001n01.meta">
+ <nvpair id="is_managed.meta.auto-104" name="is-managed" value="1"/>
+ </meta_attributes>
+ </primitive>
+ </resources>
+ <constraints>
+ <rsc_location id="run_DcIPaddr" rsc="DcIPaddr">
+ <rule id="cant_run_DcIPaddr" score="-INFINITY" boolean-op="and">
+ <expression id="expression.id22176" attribute="#is_dc" operation="eq" value="false"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="run_rsc_c001n08" rsc="rsc_c001n08">
+ <rule id="pref_run_rsc_c001n08" score="100" boolean-op="and">
+ <expression id="expression.id22204" attribute="#uname" operation="eq" value="c001n08"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="run_rsc_c001n02" rsc="rsc_c001n02">
+ <rule id="pref_run_rsc_c001n02" score="100" boolean-op="and">
+ <expression id="expression.id22231" attribute="#uname" operation="eq" value="c001n02"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="run_rsc_c001n03" rsc="rsc_c001n03">
+ <rule id="pref_run_rsc_c001n03" score="100" boolean-op="and">
+ <expression id="expression.id22259" attribute="#uname" operation="eq" value="c001n03"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="run_rsc_c001n01" rsc="rsc_c001n01">
+ <rule id="pref_run_rsc_c001n01" score="100" boolean-op="and">
+ <expression id="expression.id22286" attribute="#uname" operation="eq" value="c001n01"/>
+ </rule>
+ </rsc_location>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="6427cb5a-c7a5-4bdf-9892-a04ce56f4e6b" uname="c001n08" in_ccm="true" crmd="online" ha="active" expected="member" join="member">
+ <lrm id="lrm.auto-1">
+ <lrm_resources id="lrm_resources.id22327">
+ <lrm_resource id="rsc_c001n08" type="IPaddr" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="rsc_c001n08_start_0" operation="start" interval="0" transition-key="0:0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-status="0" call-id="2" rc-code="0" crm_feature_set="2.0" transition-magic="0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-digest="fd54394d262d771b8cb48f31f8df7242"/>
+ <lrm_rsc_op id="rsc_c001n08_monitor_5000" operation="monitor" transition-key="0:0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-status="0" call-id="3" rc-code="0" crm_feature_set="2.0" transition-magic="0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-digest="6d0c707e8887b775eee00abb63b80a67" interval="5000"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="e9bdfde9-01b0-421f-acd8-8a65a53e775f" uname="c001n02" crmd="online" in_ccm="true" ha="active" expected="member" join="member">
+ <lrm id="lrm.auto-2">
+ <lrm_resources id="lrm_resources.id22420">
+ <lrm_resource id="DcIPaddr" type="IPaddr" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="DcIPaddr_start_0" operation="start" interval="0" transition-key="0:0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-status="-1" call-id="-1" rc-code="-1" crm_feature_set="2.0" transition-magic="0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-digest="21ccbd2b8e65014ca7b36a2070642a2e" op-force-restart=" dummy " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="rsc_c001n02" type="IPaddr" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="rsc_c001n02_start_0" operation="start" interval="0" transition-key="0:0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-status="0" call-id="5" rc-code="0" crm_feature_set="2.0" transition-magic="0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-digest="d8eeae5f6724dfcab8f57ef54367eb0a"/>
+ <lrm_rsc_op id="rsc_c001n02_monitor_5000" operation="monitor" transition-key="0:0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-status="0" call-id="6" rc-code="0" crm_feature_set="2.0" transition-magic="0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-digest="7515405113757a8bc37db755be497fca" interval="5000"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="5d9a8c11-8684-43ea-91.0.6e221530c193" uname="c001n03" crmd="online" in_ccm="true" ha="active" expected="member" join="member">
+ <lrm id="lrm.auto-3">
+ <lrm_resources id="lrm_resources.id22579">
+ <lrm_resource id="rsc_c001n03" type="IPaddr" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="rsc_c001n03_start_0" operation="start" interval="0" transition-key="0:0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-status="0" call-id="3" rc-code="0" crm_feature_set="2.0" transition-magic="0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-digest="6e0f48b04277dfed400d2ffce5b5ae51"/>
+ <lrm_rsc_op id="rsc_c001n03_monitor_5000" operation="monitor" transition-key="0:0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-status="0" call-id="4" rc-code="0" crm_feature_set="2.0" transition-magic="0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-digest="3b337ce52a3a88fb0a61ff74c5e42f0b" interval="5000"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="de937e3d-0309-4b5d-b85c-f96edc1ed8e3" uname="c001n01" crmd="online" in_ccm="true" ha="active" expected="member" join="member">
+ <lrm id="lrm.auto-4">
+ <lrm_resources id="lrm_resources.id22666">
+ <lrm_resource id="rsc_c001n01" type="IPaddr" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="rsc_c001n01_start_0" operation="start" interval="0" transition-key="0:0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-status="0" call-id="2" rc-code="0" crm_feature_set="2.0" transition-magic="0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-digest="74307acd7ffa84f71fe8a46100cdbbb0"/>
+ <lrm_rsc_op id="rsc_c001n01_monitor_5000" operation="monitor" transition-key="0:0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-status="0" call-id="3" rc-code="0" crm_feature_set="2.0" transition-magic="0:0:0fbef83f-5f7a-4928-b803-3cdca478e889" op-digest="fc615675937160e95e360e325cd67d82" interval="5000"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ </status>
+</cib>
--
1.8.3.1
From ff6aebecf8b40b882bddbd0d78e3f8702f97147e Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Mon, 13 Apr 2020 12:22:35 -0500
Subject: [PATCH 06/20] Doc: libpacemaker: improve comments when logging
actions
... with slight refactoring for consistency
---
lib/pacemaker/pcmk_sched_native.c | 41 ++++++++++++++++++++++-----------------
1 file changed, 23 insertions(+), 18 deletions(-)
diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c
index ff2fb92..f14e690 100644
--- a/lib/pacemaker/pcmk_sched_native.c
+++ b/lib/pacemaker/pcmk_sched_native.c
@@ -2348,8 +2348,6 @@ native_expand(pe_resource_t * rsc, pe_working_set_t * data_set)
} \
} while(0)
-static int rsc_width = 5;
-static int detail_width = 5;
static void
LogAction(const char *change, pe_resource_t *rsc, pe_node_t *origin, pe_node_t *destination, pe_action_t *action, pe_action_t *source, gboolean terminal)
{
@@ -2360,6 +2358,9 @@ LogAction(const char *change, pe_resource_t *rsc, pe_node_t *origin, pe_node_t *
bool same_role = FALSE;
bool need_role = FALSE;
+ static int rsc_width = 5;
+ static int detail_width = 5;
+
CRM_ASSERT(action);
CRM_ASSERT(destination != NULL || origin != NULL);
@@ -2384,36 +2385,40 @@ LogAction(const char *change, pe_resource_t *rsc, pe_node_t *origin, pe_node_t *
same_role = TRUE;
}
- if(need_role && origin == NULL) {
- /* Promoting from Stopped */
+ if (need_role && (origin == NULL)) {
+ /* Starting and promoting a promotable clone instance */
details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), destination->details->uname);
- } else if(need_role && destination == NULL) {
- /* Demoting a Master or Stopping a Slave */
+ } else if (origin == NULL) {
+ /* Starting a resource */
+ details = crm_strdup_printf("%s", destination->details->uname);
+
+ } else if (need_role && (destination == NULL)) {
+ /* Stopping a promotable clone instance */
details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname);
- } else if(origin == NULL || destination == NULL) {
- /* Starting or stopping a resource */
- details = crm_strdup_printf("%s", origin?origin->details->uname:destination->details->uname);
+ } else if (destination == NULL) {
+ /* Stopping a resource */
+ details = crm_strdup_printf("%s", origin->details->uname);
- } else if(need_role && same_role && same_host) {
- /* Recovering or restarting a promotable clone resource */
+ } else if (need_role && same_role && same_host) {
+ /* Recovering, restarting or re-promoting a promotable clone instance */
details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname);
- } else if(same_role && same_host) {
+ } else if (same_role && same_host) {
/* Recovering or Restarting a normal resource */
details = crm_strdup_printf("%s", origin->details->uname);
- } else if(same_role && need_role) {
- /* Moving a promotable clone resource */
+ } else if (need_role && same_role) {
+ /* Moving a promotable clone instance */
details = crm_strdup_printf("%s -> %s %s", origin->details->uname, destination->details->uname, role2text(rsc->role));
- } else if(same_role) {
+ } else if (same_role) {
/* Moving a normal resource */
details = crm_strdup_printf("%s -> %s", origin->details->uname, destination->details->uname);
- } else if(same_host) {
- /* Promoting or demoting a promotable clone resource */
+ } else if (same_host) {
+ /* Promoting or demoting a promotable clone instance */
details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), origin->details->uname);
} else {
@@ -2560,7 +2565,7 @@ LogActions(pe_resource_t * rsc, pe_working_set_t * data_set, gboolean terminal)
pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id, role2text(rsc->role),
next->details->uname);
- } else if (start && is_set(start->flags, pe_action_runnable) == FALSE) {
+ } else if (is_not_set(start->flags, pe_action_runnable)) {
LogAction("Stop", rsc, current, NULL, stop,
(stop && stop->reason)? stop : start, terminal);
STOP_SANITY_ASSERT(__LINE__);
--
1.8.3.1
From 98c3b649fa065b7e7a59029cc2f887bc462d170a Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Mon, 13 Apr 2020 12:23:22 -0500
Subject: [PATCH 07/20] Log: libpacemaker: check for re-promotes specifically
If a promotable clone instance is being demoted and promoted on its current
node, without also stopping and starting, it previously would be logged as
"Leave" indicating unchanged, because the current and next role are the same.
Now, check for this situation specifically, and log it as "Re-promote".
Currently, the scheduler is not capable of generating this situation, but
upcoming changes will.
---
lib/pacemaker/pcmk_sched_native.c | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c
index f14e690..89952bf 100644
--- a/lib/pacemaker/pcmk_sched_native.c
+++ b/lib/pacemaker/pcmk_sched_native.c
@@ -2561,9 +2561,17 @@ LogActions(pe_resource_t * rsc, pe_working_set_t * data_set, gboolean terminal)
} else if (is_set(rsc->flags, pe_rsc_reload)) {
LogAction("Reload", rsc, current, next, start, NULL, terminal);
+
} else if (start == NULL || is_set(start->flags, pe_action_optional)) {
- pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id, role2text(rsc->role),
- next->details->uname);
+ if ((demote != NULL) && (promote != NULL)
+ && is_not_set(demote->flags, pe_action_optional)
+ && is_not_set(promote->flags, pe_action_optional)) {
+ LogAction("Re-promote", rsc, current, next, promote, demote,
+ terminal);
+ } else {
+ pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id,
+ role2text(rsc->role), next->details->uname);
+ }
} else if (is_not_set(start->flags, pe_action_runnable)) {
LogAction("Stop", rsc, current, NULL, stop,
--
1.8.3.1
From fd55a6660574c0bca517fd519377340712fb443a Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Mon, 13 Apr 2020 12:51:03 -0500
Subject: [PATCH 08/20] Doc: libpacemaker: improve comments for resource state
and action matrices
Also, make them static, for linker efficiency.
---
lib/pacemaker/pcmk_sched_native.c | 39 ++++++++++++++++++++++++---------------
1 file changed, 24 insertions(+), 15 deletions(-)
diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c
index 89952bf..b9bca80 100644
--- a/lib/pacemaker/pcmk_sched_native.c
+++ b/lib/pacemaker/pcmk_sched_native.c
@@ -41,27 +41,36 @@ gboolean PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional,
gboolean RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
gboolean NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
-/* *INDENT-OFF* */
-enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
-/* Current State */
-/* Next State: Unknown Stopped Started Slave Master */
+/* This array says what the *next* role should be when transitioning from one
+ * role to another. For example going from Stopped to Master, the next role is
+ * RSC_ROLE_SLAVE, because the resource must be started before being promoted.
+ * The current state then becomes Started, which is fed into this array again,
+ * giving a next role of RSC_ROLE_MASTER.
+ */
+static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
+ /* Current state Next state*/
+ /* Unknown Stopped Started Slave Master */
/* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, },
/* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, },
/* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
- /* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
- /* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
+ /* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
+ /* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
};
-gboolean (*rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX])(pe_resource_t*,pe_node_t*,gboolean,pe_working_set_t*) = {
-/* Current State */
-/* Next State: Unknown Stopped Started Slave Master */
- /* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, },
- /* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, },
- /* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, },
- /* Slave */ { RoleError, StopRsc, StopRsc, NullOp, PromoteRsc, },
- /* Master */ { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp, },
+typedef gboolean (*rsc_transition_fn)(pe_resource_t *rsc, pe_node_t *next,
+ gboolean optional,
+ pe_working_set_t *data_set);
+
+// This array picks the function needed to transition from one role to another
+static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
+ /* Current state Next state */
+ /* Unknown Stopped Started Slave Master */
+ /* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, },
+ /* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, },
+ /* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, },
+ /* Slave */ { RoleError, StopRsc, StopRsc, NullOp, PromoteRsc, },
+ /* Master */ { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp , },
};
-/* *INDENT-ON* */
static gboolean
native_choose_node(pe_resource_t * rsc, pe_node_t * prefer, pe_working_set_t * data_set)
--
1.8.3.1
From 2f1e2df1f5ec67591cddf14f9dda1c52919dd53a Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 26 May 2020 17:50:48 -0500
Subject: [PATCH 09/20] Feature: xml: add on-fail="demote" option to resources
schema
We don't need an XML schema version bump because it was already bumped since
the last release, for the rsc_expression/op_expression feature.
---
xml/resources-3.4.rng | 1 +
1 file changed, 1 insertion(+)
diff --git a/xml/resources-3.4.rng b/xml/resources-3.4.rng
index fbb4b65..887dc1c 100644
--- a/xml/resources-3.4.rng
+++ b/xml/resources-3.4.rng
@@ -388,6 +388,7 @@
<choice>
<value>ignore</value>
<value>block</value>
+ <value>demote</value>
<value>stop</value>
<value>restart</value>
<value>standby</value>
--
1.8.3.1
From 874f75e0faad91c634860221d727e51e95d97f19 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Thu, 28 May 2020 08:29:37 -0500
Subject: [PATCH 10/20] Feature: scheduler: new on-fail="demote" recovery
policy for promoted resources
---
include/crm/pengine/pe_types.h | 1 +
lib/pacemaker/pcmk_sched_native.c | 25 +++++++++++++++----
lib/pengine/common.c | 3 +++
lib/pengine/unpack.c | 51 ++++++++++++++++++++++++++++++++++++---
lib/pengine/utils.c | 35 +++++++++++++++++++++++----
5 files changed, 102 insertions(+), 13 deletions(-)
diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h
index ba88491..ed5eb12 100644
--- a/include/crm/pengine/pe_types.h
+++ b/include/crm/pengine/pe_types.h
@@ -246,6 +246,7 @@ struct pe_node_s {
# define pe_rsc_allocating 0x00000200ULL
# define pe_rsc_merging 0x00000400ULL
+# define pe_rsc_stop 0x00001000ULL
# define pe_rsc_reload 0x00002000ULL
# define pe_rsc_allow_remote_remotes 0x00004000ULL
diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c
index b9bca80..4e3bd7c 100644
--- a/lib/pacemaker/pcmk_sched_native.c
+++ b/lib/pacemaker/pcmk_sched_native.c
@@ -1205,6 +1205,7 @@ native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
pe_node_t *chosen = NULL;
pe_node_t *current = NULL;
gboolean need_stop = FALSE;
+ bool need_promote = FALSE;
gboolean is_moving = FALSE;
gboolean allow_migrate = is_set(rsc->flags, pe_rsc_allow_migrate) ? TRUE : FALSE;
@@ -1309,8 +1310,15 @@ native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
need_stop = TRUE;
} else if (is_set(rsc->flags, pe_rsc_failed)) {
- pe_rsc_trace(rsc, "Recovering %s", rsc->id);
- need_stop = TRUE;
+ if (is_set(rsc->flags, pe_rsc_stop)) {
+ need_stop = TRUE;
+ pe_rsc_trace(rsc, "Recovering %s", rsc->id);
+ } else {
+ pe_rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
+ if (rsc->next_role == RSC_ROLE_MASTER) {
+ need_promote = TRUE;
+ }
+ }
} else if (is_set(rsc->flags, pe_rsc_block)) {
pe_rsc_trace(rsc, "Block %s", rsc->id);
@@ -1344,10 +1352,16 @@ native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
while (rsc->role <= rsc->next_role && role != rsc->role && is_not_set(rsc->flags, pe_rsc_block)) {
+ bool required = need_stop;
+
next_role = rsc_state_matrix[role][rsc->role];
+ if ((next_role == RSC_ROLE_MASTER) && need_promote) {
+ required = true;
+ }
pe_rsc_trace(rsc, "Up: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role),
- rsc->id, need_stop ? " required" : "");
- if (rsc_action_matrix[role][next_role] (rsc, chosen, !need_stop, data_set) == FALSE) {
+ rsc->id, (required? " required" : ""));
+ if (rsc_action_matrix[role][next_role](rsc, chosen, !required,
+ data_set) == FALSE) {
break;
}
role = next_role;
@@ -2631,7 +2645,8 @@ LogActions(pe_resource_t * rsc, pe_working_set_t * data_set, gboolean terminal)
free(key);
- } else if (stop && is_set(rsc->flags, pe_rsc_failed)) {
+ } else if (stop && is_set(rsc->flags, pe_rsc_failed)
+ && is_set(rsc->flags, pe_rsc_stop)) {
/* 'stop' may be NULL if the failure was ignored */
LogAction("Recover", rsc, current, next, stop, start, terminal);
STOP_SANITY_ASSERT(__LINE__);
diff --git a/lib/pengine/common.c b/lib/pengine/common.c
index ded6df8..f4f2106 100644
--- a/lib/pengine/common.c
+++ b/lib/pengine/common.c
@@ -326,6 +326,9 @@ fail2text(enum action_fail_response fail)
case action_fail_ignore:
result = "ignore";
break;
+ case action_fail_demote:
+ result = "demote";
+ break;
case action_fail_block:
result = "block";
break;
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index 6a350e5..a219805 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -108,6 +108,7 @@ pe_fence_node(pe_working_set_t * data_set, pe_node_t * node,
*/
node->details->remote_requires_reset = TRUE;
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
}
}
@@ -117,6 +118,7 @@ pe_fence_node(pe_working_set_t * data_set, pe_node_t * node,
"and guest resource no longer exists",
node->details->uname, reason);
set_bit(node->details->remote_rsc->flags, pe_rsc_failed);
+ set_bit(node->details->remote_rsc->flags, pe_rsc_stop);
} else if (pe__is_remote_node(node)) {
pe_resource_t *rsc = node->details->remote_rsc;
@@ -1914,6 +1916,7 @@ process_rsc_state(pe_resource_t * rsc, pe_node_t * node,
*/
if (pe__is_guest_node(node)) {
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
should_fence = TRUE;
} else if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
@@ -1956,6 +1959,11 @@ process_rsc_state(pe_resource_t * rsc, pe_node_t * node,
/* nothing to do */
break;
+ case action_fail_demote:
+ set_bit(rsc->flags, pe_rsc_failed);
+ demote_action(rsc, node, FALSE);
+ break;
+
case action_fail_fence:
/* treat it as if it is still running
* but also mark the node as unclean
@@ -1992,12 +2000,14 @@ process_rsc_state(pe_resource_t * rsc, pe_node_t * node,
case action_fail_recover:
if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
stop_action(rsc, node, FALSE);
}
break;
case action_fail_restart_container:
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
if (rsc->container && pe_rsc_is_bundled(rsc)) {
/* A bundle's remote connection can run on a different node than
@@ -2016,6 +2026,7 @@ process_rsc_state(pe_resource_t * rsc, pe_node_t * node,
case action_fail_reset_remote:
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
tmpnode = NULL;
if (rsc->is_remote_node) {
@@ -2071,8 +2082,17 @@ process_rsc_state(pe_resource_t * rsc, pe_node_t * node,
}
native_add_running(rsc, node, data_set);
- if (on_fail != action_fail_ignore) {
- set_bit(rsc->flags, pe_rsc_failed);
+ switch (on_fail) {
+ case action_fail_ignore:
+ break;
+ case action_fail_demote:
+ case action_fail_block:
+ set_bit(rsc->flags, pe_rsc_failed);
+ break;
+ default:
+ set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
+ break;
}
} else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
@@ -2595,6 +2615,7 @@ unpack_migrate_to_success(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
} else {
/* Consider it failed here - forces a restart, prevents migration */
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
clear_bit(rsc->flags, pe_rsc_allow_migrate);
}
}
@@ -2785,9 +2806,21 @@ static int
cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
{
switch (first) {
+ case action_fail_demote:
+ switch (second) {
+ case action_fail_ignore:
+ return 1;
+ case action_fail_demote:
+ return 0;
+ default:
+ return -1;
+ }
+ break;
+
case action_fail_reset_remote:
switch (second) {
case action_fail_ignore:
+ case action_fail_demote:
case action_fail_recover:
return 1;
case action_fail_reset_remote:
@@ -2800,6 +2833,7 @@ cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
case action_fail_restart_container:
switch (second) {
case action_fail_ignore:
+ case action_fail_demote:
case action_fail_recover:
case action_fail_reset_remote:
return 1;
@@ -2814,9 +2848,13 @@ cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
break;
}
switch (second) {
+ case action_fail_demote:
+ return (first == action_fail_ignore)? -1 : 1;
+
case action_fail_reset_remote:
switch (first) {
case action_fail_ignore:
+ case action_fail_demote:
case action_fail_recover:
return -1;
default:
@@ -2827,6 +2865,7 @@ cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
case action_fail_restart_container:
switch (first) {
case action_fail_ignore:
+ case action_fail_demote:
case action_fail_recover:
case action_fail_reset_remote:
return -1;
@@ -3426,7 +3465,11 @@ update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, c
clear_past_failure = TRUE;
} else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
- /* Demote from Master does not clear an error */
+
+ if (*on_fail == action_fail_demote) {
+ // Demote clears an error only if on-fail=demote
+ clear_past_failure = TRUE;
+ }
rsc->role = RSC_ROLE_SLAVE;
} else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
@@ -3454,6 +3497,7 @@ update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, c
case action_fail_block:
case action_fail_ignore:
+ case action_fail_demote:
case action_fail_recover:
case action_fail_restart_container:
*on_fail = action_fail_ignore;
@@ -3714,6 +3758,7 @@ unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
* that, ensure the remote connection is considered failed.
*/
set_bit(node->details->remote_rsc->flags, pe_rsc_failed);
+ set_bit(node->details->remote_rsc->flags, pe_rsc_stop);
}
// fall through
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index 3fb7e62..fee9efb 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -720,6 +720,7 @@ static bool
valid_stop_on_fail(const char *value)
{
return safe_str_neq(value, "standby")
+ && safe_str_neq(value, "demote")
&& safe_str_neq(value, "stop");
}
@@ -727,6 +728,11 @@ static const char *
unpack_operation_on_fail(pe_action_t * action)
{
+ const char *name = NULL;
+ const char *role = NULL;
+ const char *on_fail = NULL;
+ const char *interval_spec = NULL;
+ const char *enabled = NULL;
const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);
if (safe_str_eq(action->task, CRMD_ACTION_STOP)
@@ -736,14 +742,10 @@ unpack_operation_on_fail(pe_action_t * action)
"action to default value because '%s' is not "
"allowed for stop", action->rsc->id, value);
return NULL;
+
} else if (safe_str_eq(action->task, CRMD_ACTION_DEMOTE) && !value) {
/* demote on_fail defaults to master monitor value if present */
xmlNode *operation = NULL;
- const char *name = NULL;
- const char *role = NULL;
- const char *on_fail = NULL;
- const char *interval_spec = NULL;
- const char *enabled = NULL;
CRM_CHECK(action->rsc != NULL, return NULL);
@@ -766,12 +768,31 @@ unpack_operation_on_fail(pe_action_t * action)
continue;
} else if (crm_parse_interval_spec(interval_spec) == 0) {
continue;
+ } else if (safe_str_eq(on_fail, "demote")) {
+ continue;
}
value = on_fail;
}
} else if (safe_str_eq(action->task, CRM_OP_LRM_DELETE)) {
value = "ignore";
+
+ } else if (safe_str_eq(value, "demote")) {
+ name = crm_element_value(action->op_entry, "name");
+ role = crm_element_value(action->op_entry, "role");
+ on_fail = crm_element_value(action->op_entry, XML_OP_ATTR_ON_FAIL);
+ interval_spec = crm_element_value(action->op_entry,
+ XML_LRM_ATTR_INTERVAL);
+
+ if (safe_str_neq(name, CRMD_ACTION_PROMOTE)
+ && (safe_str_neq(name, CRMD_ACTION_STATUS)
+ || safe_str_neq(role, "Master")
+ || (crm_parse_interval_spec(interval_spec) == 0))) {
+ pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s %s "
+ "action to default value because 'demote' is not "
+ "allowed for it", action->rsc->id, name);
+ return NULL;
+ }
}
return value;
@@ -1170,6 +1191,10 @@ unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * contai
value = NULL;
}
+ } else if (safe_str_eq(value, "demote")) {
+ action->on_fail = action_fail_demote;
+ value = "demote instance";
+
} else {
pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value);
value = NULL;
--
1.8.3.1
From d29433ea57796de000f4fea8c60f8da1d903108b Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 16 Jun 2020 16:03:14 -0500
Subject: [PATCH 11/20] Test: scheduler: add regression tests for
on-fail="demote"
---
cts/cts-scheduler.in | 4 +
cts/scheduler/on_fail_demote1.dot | 64 ++
cts/scheduler/on_fail_demote1.exp | 360 +++++++
cts/scheduler/on_fail_demote1.scores | 470 +++++++++
cts/scheduler/on_fail_demote1.summary | 86 ++
cts/scheduler/on_fail_demote1.xml | 616 +++++++++++
cts/scheduler/on_fail_demote2.dot | 22 +
cts/scheduler/on_fail_demote2.exp | 125 +++
cts/scheduler/on_fail_demote2.scores | 127 +++
cts/scheduler/on_fail_demote2.summary | 41 +
cts/scheduler/on_fail_demote2.xml | 221 ++++
cts/scheduler/on_fail_demote3.dot | 12 +
cts/scheduler/on_fail_demote3.exp | 63 ++
cts/scheduler/on_fail_demote3.scores | 127 +++
cts/scheduler/on_fail_demote3.summary | 34 +
cts/scheduler/on_fail_demote3.xml | 221 ++++
cts/scheduler/on_fail_demote4.dot | 383 +++++++
cts/scheduler/on_fail_demote4.exp | 1818 +++++++++++++++++++++++++++++++++
cts/scheduler/on_fail_demote4.scores | 470 +++++++++
cts/scheduler/on_fail_demote4.summary | 187 ++++
cts/scheduler/on_fail_demote4.xml | 625 ++++++++++++
21 files changed, 6076 insertions(+)
create mode 100644 cts/scheduler/on_fail_demote1.dot
create mode 100644 cts/scheduler/on_fail_demote1.exp
create mode 100644 cts/scheduler/on_fail_demote1.scores
create mode 100644 cts/scheduler/on_fail_demote1.summary
create mode 100644 cts/scheduler/on_fail_demote1.xml
create mode 100644 cts/scheduler/on_fail_demote2.dot
create mode 100644 cts/scheduler/on_fail_demote2.exp
create mode 100644 cts/scheduler/on_fail_demote2.scores
create mode 100644 cts/scheduler/on_fail_demote2.summary
create mode 100644 cts/scheduler/on_fail_demote2.xml
create mode 100644 cts/scheduler/on_fail_demote3.dot
create mode 100644 cts/scheduler/on_fail_demote3.exp
create mode 100644 cts/scheduler/on_fail_demote3.scores
create mode 100644 cts/scheduler/on_fail_demote3.summary
create mode 100644 cts/scheduler/on_fail_demote3.xml
create mode 100644 cts/scheduler/on_fail_demote4.dot
create mode 100644 cts/scheduler/on_fail_demote4.exp
create mode 100644 cts/scheduler/on_fail_demote4.scores
create mode 100644 cts/scheduler/on_fail_demote4.summary
create mode 100644 cts/scheduler/on_fail_demote4.xml
diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
index ae8247e..0e68e73 100644
--- a/cts/cts-scheduler.in
+++ b/cts/cts-scheduler.in
@@ -478,6 +478,10 @@ TESTS = [
[ "master-score-startup", "Use permanent master scores without LRM history" ],
[ "failed-demote-recovery", "Recover resource in slave role after demote fails" ],
[ "failed-demote-recovery-master", "Recover resource in master role after demote fails" ],
+ [ "on_fail_demote1", "Recovery with on-fail=\"demote\" on healthy cluster, remote, guest, and bundle nodes" ],
+ [ "on_fail_demote2", "Recovery with on-fail=\"demote\" with promotion on different node" ],
+ [ "on_fail_demote3", "Recovery with on-fail=\"demote\" with no promotion" ],
+ [ "on_fail_demote4", "Recovery with on-fail=\"demote\" on failed cluster, remote, guest, and bundle nodes" ],
],
[
[ "history-1", "Correctly parse stateful-1 resource state" ],
diff --git a/cts/scheduler/on_fail_demote1.dot b/cts/scheduler/on_fail_demote1.dot
new file mode 100644
index 0000000..d11c1c1
--- /dev/null
+++ b/cts/scheduler/on_fail_demote1.dot
@@ -0,0 +1,64 @@
+ digraph "g" {
+"bundled_demote_0 stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"bundled_demote_0 stateful-bundle-0" -> "stateful-bundle-master_demoted_0" [ style = bold]
+"bundled_demote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"]
+"bundled_promote_0 stateful-bundle-0" -> "stateful-bundle-master_promoted_0" [ style = bold]
+"bundled_promote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"]
+"lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold]
+"lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc2" [ style = bold]
+"lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold]
+"lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms_demote_0 lxc2" -> "lxc-ms-master_demoted_0" [ style = bold]
+"lxc-ms_demote_0 lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc-ms_demote_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"lxc-ms_promote_0 lxc2" -> "lxc-ms-master_promoted_0" [ style = bold]
+"lxc-ms_promote_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_demoted_0" -> "rsc1-clone_promote_0" [ style = bold]
+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promote_0" -> "rsc1_promote_0 rhel7-4" [ style = bold]
+"rsc1-clone_promote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1_demote_0 rhel7-4" -> "rsc1_promote_0 rhel7-4" [ style = bold]
+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc1_promote_0 rhel7-4" -> "rsc1-clone_promoted_0" [ style = bold]
+"rsc1_promote_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc2-master_demote_0" -> "rsc2-master_demoted_0" [ style = bold]
+"rsc2-master_demote_0" -> "rsc2_demote_0 remote-rhel7-2" [ style = bold]
+"rsc2-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_demoted_0" -> "rsc2-master_promote_0" [ style = bold]
+"rsc2-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_promote_0" -> "rsc2_promote_0 remote-rhel7-2" [ style = bold]
+"rsc2-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2_demote_0 remote-rhel7-2" -> "rsc2-master_demoted_0" [ style = bold]
+"rsc2_demote_0 remote-rhel7-2" -> "rsc2_promote_0 remote-rhel7-2" [ style = bold]
+"rsc2_demote_0 remote-rhel7-2" [ style=bold color="green" fontcolor="black"]
+"rsc2_promote_0 remote-rhel7-2" -> "rsc2-master_promoted_0" [ style = bold]
+"rsc2_promote_0 remote-rhel7-2" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-master_demote_0" -> "bundled_demote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_demote_0" -> "stateful-bundle-master_demoted_0" [ style = bold]
+"stateful-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle_demoted_0" [ style = bold]
+"stateful-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_promote_0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_promoted_0" -> "stateful-bundle_promoted_0" [ style = bold]
+"stateful-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_demote_0" -> "stateful-bundle-master_demote_0" [ style = bold]
+"stateful-bundle_demote_0" -> "stateful-bundle_demoted_0" [ style = bold]
+"stateful-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_demoted_0" -> "stateful-bundle_promote_0" [ style = bold]
+"stateful-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_promote_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/cts/scheduler/on_fail_demote1.exp b/cts/scheduler/on_fail_demote1.exp
new file mode 100644
index 0000000..ebe1dd5
--- /dev/null
+++ b/cts/scheduler/on_fail_demote1.exp
@@ -0,0 +1,360 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="1" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="42" operation="promote" operation_key="rsc1_promote_0" internal_operation_key="rsc1:0_promote_0" on_node="rhel7-4" on_node_uuid="4">
+ <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="3" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="59" operation="promote" operation_key="rsc1-clone_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="3" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-4" on_node_uuid="4">
+ <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="61" operation="demote" operation_key="rsc1-clone_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2" priority="1000000">
+ <action_set>
+ <pseudo_event id="62" operation="demoted" operation_key="rsc1-clone_demoted_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="3" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="61" operation="demote" operation_key="rsc1-clone_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <pseudo_event id="61" operation="demote" operation_key="rsc1-clone_demote_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+ <pseudo_event id="60" operation="promoted" operation_key="rsc1-clone_promoted_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="42" operation="promote" operation_key="rsc1_promote_0" internal_operation_key="rsc1:0_promote_0" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <pseudo_event id="59" operation="promote" operation_key="rsc1-clone_promote_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="62" operation="demoted" operation_key="rsc1-clone_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <rsc_op id="73" operation="promote" operation_key="rsc2_promote_0" internal_operation_key="rsc2:4_promote_0" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2" router_node="rhel7-1">
+ <primitive id="rsc2" long-id="rsc2:4" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="4" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="remote-rhel7-2" CRM_meta_on_node_uuid="remote-rhel7-2" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="25" operation="demote" operation_key="rsc2_demote_0" internal_operation_key="rsc2:4_demote_0" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2" router_node="rhel7-1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="82" operation="promote" operation_key="rsc2-master_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <rsc_op id="25" operation="demote" operation_key="rsc2_demote_0" internal_operation_key="rsc2:4_demote_0" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2" router_node="rhel7-1">
+ <primitive id="rsc2" long-id="rsc2:4" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="4" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="remote-rhel7-2" CRM_meta_on_node_uuid="remote-rhel7-2" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="84" operation="demote" operation_key="rsc2-master_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8" priority="1000000">
+ <action_set>
+ <pseudo_event id="85" operation="demoted" operation_key="rsc2-master_demoted_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="25" operation="demote" operation_key="rsc2_demote_0" internal_operation_key="rsc2:4_demote_0" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2" router_node="rhel7-1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="84" operation="demote" operation_key="rsc2-master_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <pseudo_event id="84" operation="demote" operation_key="rsc2-master_demote_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="10" priority="1000000">
+ <action_set>
+ <pseudo_event id="83" operation="promoted" operation_key="rsc2-master_promoted_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="73" operation="promote" operation_key="rsc2_promote_0" internal_operation_key="rsc2:4_promote_0" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2" router_node="rhel7-1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="11">
+ <action_set>
+ <pseudo_event id="82" operation="promote" operation_key="rsc2-master_promote_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="85" operation="demoted" operation_key="rsc2-master_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="12">
+ <action_set>
+ <rsc_op id="94" operation="promote" operation_key="lxc-ms_promote_0" internal_operation_key="lxc-ms:0_promote_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3">
+ <primitive id="lxc-ms" long-id="lxc-ms:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="lxc2" CRM_meta_on_node_uuid="lxc2" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="29" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="101" operation="promote" operation_key="lxc-ms-master_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="13">
+ <action_set>
+ <rsc_op id="29" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3">
+ <primitive id="lxc-ms" long-id="lxc-ms:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="lxc2" CRM_meta_on_node_uuid="lxc2" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="103" operation="demote" operation_key="lxc-ms-master_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="14" priority="1000000">
+ <action_set>
+ <pseudo_event id="104" operation="demoted" operation_key="lxc-ms-master_demoted_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="29" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="103" operation="demote" operation_key="lxc-ms-master_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="15">
+ <action_set>
+ <pseudo_event id="103" operation="demote" operation_key="lxc-ms-master_demote_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="16" priority="1000000">
+ <action_set>
+ <pseudo_event id="102" operation="promoted" operation_key="lxc-ms-master_promoted_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="94" operation="promote" operation_key="lxc-ms_promote_0" internal_operation_key="lxc-ms:0_promote_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="17">
+ <action_set>
+ <pseudo_event id="101" operation="promote" operation_key="lxc-ms-master_promote_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="104" operation="demoted" operation_key="lxc-ms-master_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="18">
+ <action_set>
+ <rsc_op id="129" operation="promote" operation_key="bundled_promote_0" internal_operation_key="bundled:0_promote_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5">
+ <primitive id="bundled" long-id="bundled:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="stateful-bundle-0" CRM_meta_on_node_uuid="stateful-bundle-0" CRM_meta_physical_host="rhel7-5" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" pcmk_external_ip="192.168.122.131"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="36" operation="demote" operation_key="bundled_demote_0" internal_operation_key="bundled:0_demote_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="138" operation="promote" operation_key="stateful-bundle-master_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="19">
+ <action_set>
+ <rsc_op id="36" operation="demote" operation_key="bundled_demote_0" internal_operation_key="bundled:0_demote_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5">
+ <primitive id="bundled" long-id="bundled:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="stateful-bundle-0" CRM_meta_on_node_uuid="stateful-bundle-0" CRM_meta_physical_host="rhel7-5" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" pcmk_external_ip="192.168.122.131"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="140" operation="demote" operation_key="stateful-bundle-master_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="20" priority="1000000">
+ <action_set>
+ <pseudo_event id="141" operation="demoted" operation_key="stateful-bundle-master_demoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="36" operation="demote" operation_key="bundled_demote_0" internal_operation_key="bundled:0_demote_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="140" operation="demote" operation_key="stateful-bundle-master_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="21">
+ <action_set>
+ <pseudo_event id="140" operation="demote" operation_key="stateful-bundle-master_demote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="144" operation="demote" operation_key="stateful-bundle_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="22" priority="1000000">
+ <action_set>
+ <pseudo_event id="139" operation="promoted" operation_key="stateful-bundle-master_promoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="129" operation="promote" operation_key="bundled_promote_0" internal_operation_key="bundled:0_promote_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="23">
+ <action_set>
+ <pseudo_event id="138" operation="promote" operation_key="stateful-bundle-master_promote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="141" operation="demoted" operation_key="stateful-bundle-master_demoted_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="142" operation="promote" operation_key="stateful-bundle_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="24" priority="1000000">
+ <action_set>
+ <pseudo_event id="145" operation="demoted" operation_key="stateful-bundle_demoted_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="141" operation="demoted" operation_key="stateful-bundle-master_demoted_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="144" operation="demote" operation_key="stateful-bundle_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="25">
+ <action_set>
+ <pseudo_event id="144" operation="demote" operation_key="stateful-bundle_demote_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="26" priority="1000000">
+ <action_set>
+ <pseudo_event id="143" operation="promoted" operation_key="stateful-bundle_promoted_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="139" operation="promoted" operation_key="stateful-bundle-master_promoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="27">
+ <action_set>
+ <pseudo_event id="142" operation="promote" operation_key="stateful-bundle_promote_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="145" operation="demoted" operation_key="stateful-bundle_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/cts/scheduler/on_fail_demote1.scores b/cts/scheduler/on_fail_demote1.scores
new file mode 100644
index 0000000..7df582f
--- /dev/null
+++ b/cts/scheduler/on_fail_demote1.scores
@@ -0,0 +1,470 @@
+Allocation scores:
+Using the original execution date of: 2020-06-16 19:23:21Z
+bundled:0 promotion score on stateful-bundle-0: 10
+bundled:1 promotion score on stateful-bundle-1: 5
+bundled:2 promotion score on stateful-bundle-2: 5
+lxc-ms:0 promotion score on lxc2: INFINITY
+lxc-ms:1 promotion score on lxc1: INFINITY
+pcmk__bundle_allocate: bundled:0 allocation score on stateful-bundle-0: 501
+pcmk__bundle_allocate: bundled:1 allocation score on stateful-bundle-1: 501
+pcmk__bundle_allocate: bundled:2 allocation score on stateful-bundle-2: 501
+pcmk__bundle_allocate: stateful-bundle allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-0: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-2: -INFINITY
+pcmk__clone_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY
+pcmk__clone_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY
+pcmk__clone_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1-clone allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:0 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:1 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:2 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc1:3 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:3 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:4 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:4 allocation score on remote-rhel7-2: 6
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:5 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:5 allocation score on lxc2: 6
+pcmk__clone_allocate: rsc1:5 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:6 allocation score on lxc1: 6
+pcmk__clone_allocate: rsc1:6 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:6 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2-master allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2-master allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2-master allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:0 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:1 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:2 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc2:3 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:3 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:4 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:4 allocation score on remote-rhel7-2: 11
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:5 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:5 allocation score on lxc2: 6
+pcmk__clone_allocate: rsc2:5 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:6 allocation score on lxc1: 6
+pcmk__clone_allocate: rsc2:6 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:6 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-5: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc1: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc2: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on remote-rhel7-2: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-1: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-3: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-4: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-5: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-0: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-1: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-2: 0
+pcmk__native_allocate: Fencing allocation score on lxc1: -INFINITY
+pcmk__native_allocate: Fencing allocation score on lxc2: -INFINITY
+pcmk__native_allocate: Fencing allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0
+pcmk__native_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY
+pcmk__native_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY
+pcmk__native_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY
+pcmk__native_allocate: container1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: container1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: container1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: container1 allocation score on rhel7-1: 0
+pcmk__native_allocate: container1 allocation score on rhel7-3: INFINITY
+pcmk__native_allocate: container1 allocation score on rhel7-4: 0
+pcmk__native_allocate: container1 allocation score on rhel7-5: 0
+pcmk__native_allocate: container2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: container2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: container2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: container2 allocation score on rhel7-1: 0
+pcmk__native_allocate: container2 allocation score on rhel7-3: INFINITY
+pcmk__native_allocate: container2 allocation score on rhel7-4: 0
+pcmk__native_allocate: container2 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on lxc2: INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-1: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-4: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on lxc1: INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on lxc2: INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-4: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-1: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-3: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-4: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:1 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:1 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:2 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:2 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:2 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc1:3 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:3 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:3 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:4 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:4 allocation score on remote-rhel7-2: 6
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:5 allocation score on lxc2: 6
+pcmk__native_allocate: rsc1:5 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on lxc1: 6
+pcmk__native_allocate: rsc1:6 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:0 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:0 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:1 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:1 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:1 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:2 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:2 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:2 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc2:3 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:3 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:3 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on remote-rhel7-2: 11
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:5 allocation score on lxc2: 6
+pcmk__native_allocate: rsc2:5 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on lxc1: 6
+pcmk__native_allocate: rsc2:6 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-5: 10000
+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-1: 10000
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-4: 10000
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: -10000
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: -10000
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: -10000
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: -INFINITY
+rsc1:0 promotion score on rhel7-4: 10
+rsc1:1 promotion score on rhel7-3: 5
+rsc1:2 promotion score on rhel7-5: 5
+rsc1:3 promotion score on rhel7-1: 5
+rsc1:4 promotion score on remote-rhel7-2: 5
+rsc1:5 promotion score on lxc2: 5
+rsc1:6 promotion score on lxc1: 5
+rsc2:0 promotion score on rhel7-4: 10
+rsc2:1 promotion score on rhel7-3: 5
+rsc2:2 promotion score on rhel7-5: 5
+rsc2:3 promotion score on rhel7-1: 5
+rsc2:4 promotion score on remote-rhel7-2: 110
+rsc2:5 promotion score on lxc2: 5
+rsc2:6 promotion score on lxc1: 5
diff --git a/cts/scheduler/on_fail_demote1.summary b/cts/scheduler/on_fail_demote1.summary
new file mode 100644
index 0000000..b173582
--- /dev/null
+++ b/cts/scheduler/on_fail_demote1.summary
@@ -0,0 +1,86 @@
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Current cluster status:
+Online: [ rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+RemoteOnline: [ remote-rhel7-2 ]
+GuestOnline: [ lxc1:container1 lxc2:container2 stateful-bundle-0:stateful-bundle-docker-0 stateful-bundle-1:stateful-bundle-docker-1 stateful-bundle-2:stateful-bundle-docker-2 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-4
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ rsc1 (ocf::pacemaker:Stateful): FAILED Master rhel7-4
+ Slaves: [ lxc1 lxc2 remote-rhel7-2 rhel7-1 rhel7-3 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ rsc2 (ocf::pacemaker:Stateful): FAILED Master remote-rhel7-2
+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+ remote-rhel7-2 (ocf::pacemaker:remote): Started rhel7-1
+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ Clone Set: lxc-ms-master [lxc-ms] (promotable)
+ lxc-ms (ocf::pacemaker:Stateful): FAILED Master lxc2
+ Slaves: [ lxc1 ]
+ Stopped: [ remote-rhel7-2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+ Container bundle set: stateful-bundle [pcmktest:http]
+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): FAILED Master rhel7-5
+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1
+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): Slave rhel7-4
+
+Transition Summary:
+ * Re-promote rsc1:0 ( Master rhel7-4 )
+ * Re-promote rsc2:4 ( Master remote-rhel7-2 )
+ * Re-promote lxc-ms:0 ( Master lxc2 )
+ * Re-promote bundled:0 ( Master stateful-bundle-0 )
+
+Executing cluster transition:
+ * Pseudo action: rsc1-clone_demote_0
+ * Pseudo action: rsc2-master_demote_0
+ * Pseudo action: lxc-ms-master_demote_0
+ * Pseudo action: stateful-bundle_demote_0
+ * Resource action: rsc1 demote on rhel7-4
+ * Pseudo action: rsc1-clone_demoted_0
+ * Pseudo action: rsc1-clone_promote_0
+ * Resource action: rsc2 demote on remote-rhel7-2
+ * Pseudo action: rsc2-master_demoted_0
+ * Pseudo action: rsc2-master_promote_0
+ * Resource action: lxc-ms demote on lxc2
+ * Pseudo action: lxc-ms-master_demoted_0
+ * Pseudo action: lxc-ms-master_promote_0
+ * Pseudo action: stateful-bundle-master_demote_0
+ * Resource action: rsc1 promote on rhel7-4
+ * Pseudo action: rsc1-clone_promoted_0
+ * Resource action: rsc2 promote on remote-rhel7-2
+ * Pseudo action: rsc2-master_promoted_0
+ * Resource action: lxc-ms promote on lxc2
+ * Pseudo action: lxc-ms-master_promoted_0
+ * Resource action: bundled demote on stateful-bundle-0
+ * Pseudo action: stateful-bundle-master_demoted_0
+ * Pseudo action: stateful-bundle_demoted_0
+ * Pseudo action: stateful-bundle_promote_0
+ * Pseudo action: stateful-bundle-master_promote_0
+ * Resource action: bundled promote on stateful-bundle-0
+ * Pseudo action: stateful-bundle-master_promoted_0
+ * Pseudo action: stateful-bundle_promoted_0
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Revised cluster status:
+Online: [ rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+RemoteOnline: [ remote-rhel7-2 ]
+GuestOnline: [ lxc1:container1 lxc2:container2 stateful-bundle-0:stateful-bundle-docker-0 stateful-bundle-1:stateful-bundle-docker-1 stateful-bundle-2:stateful-bundle-docker-2 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-4
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Masters: [ rhel7-4 ]
+ Slaves: [ lxc1 lxc2 remote-rhel7-2 rhel7-1 rhel7-3 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ remote-rhel7-2 ]
+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+ remote-rhel7-2 (ocf::pacemaker:remote): Started rhel7-1
+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ Clone Set: lxc-ms-master [lxc-ms] (promotable)
+ Masters: [ lxc2 ]
+ Slaves: [ lxc1 ]
+ Container bundle set: stateful-bundle [pcmktest:http]
+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): Master rhel7-5
+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1
+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): Slave rhel7-4
+
diff --git a/cts/scheduler/on_fail_demote1.xml b/cts/scheduler/on_fail_demote1.xml
new file mode 100644
index 0000000..9f3ff20
--- /dev/null
+++ b/cts/scheduler/on_fail_demote1.xml
@@ -0,0 +1,616 @@
+<cib crm_feature_set="3.4.0" validate-with="pacemaker-3.4" epoch="62" num_updates="98" admin_epoch="1" cib-last-written="Tue Jun 16 14:21:19 2020" update-origin="rhel7-2" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2" execution-date="1592335401">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/>
+ <nvpair id="cts-start-failure-is-fatal" name="start-failure-is-fatal" value="false"/>
+ <nvpair id="cts-pe-input-series-max" name="pe-input-series-max" value="5000"/>
+ <nvpair id="cts-shutdown-escalation" name="shutdown-escalation" value="5min"/>
+ <nvpair id="cts-batch-limit" name="batch-limit" value="10"/>
+ <nvpair id="cts-dc-deadtime" name="dc-deadtime" value="5s"/>
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-578.6e1b582.git.el7_8-6e1b582"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="mycluster"/>
+ <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1591654576"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="rhel7-1">
+ <instance_attributes id="rhel7-1-1">
+ <nvpair id="rhel7-1-1-cts-fencing" name="cts-fencing" value="levels-and"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="rhel7-3"/>
+ <node id="4" uname="rhel7-4">
+ <instance_attributes id="nodes-4">
+ <nvpair id="nodes-4-standby" name="standby" value="off"/>
+ </instance_attributes>
+ </node>
+ <node id="5" uname="rhel7-5"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="Fencing" type="fence_xvm">
+ <meta_attributes id="Fencing-meta"/>
+ <instance_attributes id="Fencing-params">
+ <nvpair id="Fencing-key_file" name="key_file" value="/etc/pacemaker/fence_xvm.key"/>
+ <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/>
+ <nvpair id="Fencing-pcmk_host_map" name="pcmk_host_map" value="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
+ <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 rhel7-3 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5"/>
+ </instance_attributes>
+ <operations>
+ <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/>
+ <op id="Fencing-stop-0" interval="0" name="stop" timeout="60s"/>
+ <op id="Fencing-start-0" interval="0" name="start" timeout="60s"/>
+ </operations>
+ </primitive>
+ <clone id="rsc1-clone">
+ <meta_attributes id="rsc1-meta_attributes">
+ <nvpair id="rsc1-meta_attributes-promotable" name="promotable" value="true"/>
+ </meta_attributes>
+ <primitive class="ocf" id="rsc1" provider="pacemaker" type="Stateful">
+ <operations>
+ <op id="rsc1-demote-interval-0s" interval="0s" name="demote" timeout="10s"/>
+ <op id="rsc1-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="rsc1-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ <op id="rsc1-notify-interval-0s" interval="0s" name="notify" timeout="5s"/>
+ <op id="rsc1-promote-interval-0s" interval="0s" name="promote" timeout="10s"/>
+ <op id="rsc1-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+ <op id="rsc1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+ </operations>
+ </primitive>
+ </clone>
+ <clone id="rsc2-master">
+ <meta_attributes id="rsc2-meta_attributes">
+ <nvpair id="rsc2-meta_attributes-promotable" name="promotable" value="true"/>
+ </meta_attributes>
+ <primitive class="ocf" id="rsc2" provider="pacemaker" type="Stateful">
+ <operations>
+ <op id="rsc2-demote-interval-0s" interval="0s" name="demote" timeout="10s"/>
+ <op id="rsc2-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="rsc2-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ <op id="rsc2-notify-interval-0s" interval="0s" name="notify" timeout="5s"/>
+ <op id="rsc2-promote-interval-0s" interval="0s" name="promote" timeout="10s"/>
+ <op id="rsc2-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+ <op id="rsc2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+ </operations>
+ </primitive>
+ </clone>
+ <primitive class="ocf" id="remote-rhel7-2" provider="pacemaker" type="remote">
+ <instance_attributes id="remote-instance_attributes">
+ <nvpair id="remote-instance_attributes-server" name="server" value="rhel7-2"/>
+ </instance_attributes>
+ <operations>
+ <op id="remote-monitor-interval-60s" interval="60s" name="monitor"/>
+ <op id="remote-name-start-interval-0-timeout-120" interval="0" name="start" timeout="120"/>
+ </operations>
+ </primitive>
+ <primitive class="ocf" id="container1" provider="heartbeat" type="VirtualDomain">
+ <instance_attributes id="container1-instance_attributes">
+ <nvpair id="container1-instance_attributes-force_stop" name="force_stop" value="true"/>
+ <nvpair id="container1-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
+ <nvpair id="container1-instance_attributes-config" name="config" value="/var/lib/pacemaker/cts/lxc/lxc1.xml"/>
+ </instance_attributes>
+ <meta_attributes id="container1-meta_attributes">
+ <nvpair id="container1-meta_attributes-remote-node" name="remote-node" value="lxc1"/>
+ </meta_attributes>
+ </primitive>
+ <primitive class="ocf" id="container2" provider="heartbeat" type="VirtualDomain">
+ <instance_attributes id="container2-instance_attributes">
+ <nvpair id="container2-instance_attributes-force_stop" name="force_stop" value="true"/>
+ <nvpair id="container2-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
+ <nvpair id="container2-instance_attributes-config" name="config" value="/var/lib/pacemaker/cts/lxc/lxc2.xml"/>
+ </instance_attributes>
+ <meta_attributes id="container2-meta_attributes">
+ <nvpair id="container2-meta_attributes-remote-node" name="remote-node" value="lxc2"/>
+ </meta_attributes>
+ </primitive>
+ <clone id="lxc-ms-master">
+ <meta_attributes id="lxc-ms-meta_attributes">
+ <nvpair id="lxc-ms-meta_attributes-promotable" name="promotable" value="true"/>
+ <nvpair id="lxc-ms-meta_attributes-master-max" name="promoted-max" value="1"/>
+ <nvpair id="lxc-ms-meta_attributes-clone-max" name="clone-max" value="2"/>
+ </meta_attributes>
+ <primitive class="ocf" id="lxc-ms" provider="pacemaker" type="Stateful">
+ <instance_attributes id="lxc-ms-instance_attributes"/>
+ <operations>
+ <op id="lxc-ms-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="lxc-ms-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ </operations>
+ </primitive>
+ </clone>
+ <bundle id="stateful-bundle">
+ <docker image="pcmktest:http" replicas="3" promoted-max="1" options="--log-driver=journald"/>
+ <network ip-range-start="192.168.122.131" host-interface="eth0" host-netmask="24" control-port="9999">
+ <port-mapping id="bundled-port" port="80"/>
+ </network>
+ <primitive class="ocf" id="bundled" provider="pacemaker" type="Stateful">
+ <operations>
+ <op id="bundled-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="bundled-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ </operations>
+ </primitive>
+ </bundle>
+ </resources>
+ <constraints>
+ <rsc_location id="location-rsc1-clone" rsc="rsc1-clone">
+ <rule id="location-rsc1-clone-rule" role="master" score="-INFINITY">
+ <expression attribute="fail-count-rsc1#monitor_10000" id="location-rsc1-clone-rule-expr" operation="gt" value="2"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="location-rsc2-master" rsc="rsc2-master">
+ <rule id="location-rsc2-master-rule" role="master" score="100">
+ <expression id="location-rsc2-master-rule-expr" attribute="#uname" operation="eq" value="remote-rhel7-2"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="cli-prefer-container1" rsc="container1" role="Started" node="rhel7-3" score="INFINITY"/>
+ <rsc_location id="cli-prefer-container2" rsc="container2" role="Started" node="rhel7-3" score="INFINITY"/>
+ <rsc_location id="lxc-ms-location-lxc1" node="lxc1" rsc="lxc-ms-master" score="INFINITY"/>
+ <rsc_location id="lxc-ms-location-lxc2" node="lxc2" rsc="lxc-ms-master" score="INFINITY"/>
+ </constraints>
+ <op_defaults>
+ <meta_attributes id="cts-op_defaults-meta">
+ <nvpair id="cts-op_defaults-timeout" name="timeout" value="90s"/>
+ </meta_attributes>
+ </op_defaults>
+ <alerts>
+ <alert id="alert-1" path="/var/lib/pacemaker/notify.sh">
+ <recipient id="alert-1-recipient-1" value="/run/crm/alert.log"/>
+ </alert>
+ </alerts>
+ </configuration>
+ <status>
+ <node_state id="4" uname="rhel7-4" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="28:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;28:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="38" rc-code="0" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="148" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="29:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;29:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="47" rc-code="0" op-status="0" interval="120000" last-rc-change="1592342392" exec-time="65" queue-time="0" op-digest="cb34bc19df153021ce8f301baa293f35"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="6:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;6:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="18" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="94" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_10000" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="12:2:8:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:8;12:2:8:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="20" rc-code="8" op-status="0" interval="10000" last-rc-change="1592335281" exec-time="26" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="rsc1_last_failure_0" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="12:2:8:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;12:2:8:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="20" rc-code="7" op-status="0" interval="10000" last-rc-change="1592335401" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_demote_0" operation="demote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="37:0:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" transition-magic="0:0;37:0:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" exit-reason="" on_node="rhel7-4" call-id="49" rc-code="0" op-status="0" interval="0" last-rc-change="1592340727" last-run="1592340727" exec-time="55" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="36:1:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" transition-magic="0:0;36:1:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" exit-reason="" on_node="rhel7-4" call-id="51" rc-code="0" op-status="0" interval="11000" last-rc-change="1592340727" exec-time="19" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="remote-rhel7-2" type="remote" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="remote-rhel7-2_last_0" operation_key="remote-rhel7-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="14:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:7;14:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-4" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1592340505" last-run="1592340505" exec-time="0" queue-time="0" op-digest="f1c162fde9f781a7f852b761a6e079e5" op-force-restart=" server " op-restart-digest="f1c162fde9f781a7f852b761a6e079e5"/>
+ </lrm_resource>
+ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="24:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;24:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="33" rc-code="7" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="547" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
+ </lrm_resource>
+ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="25:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;25:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="37" rc-code="7" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="527" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
+ </lrm_resource>
+ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
+ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="24:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;24:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="3" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
+ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="23:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;23:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="2" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="22:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;22:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="43" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="123" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-0" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-0">
+ <lrm_rsc_op id="stateful-bundle-0_last_0" operation_key="stateful-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="54:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;54:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="10" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="0" queue-time="0" op-digest="bd3b59fd6cf7aabf0f8a6eed8a0e2a71" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-2" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-2">
+ <lrm_rsc_op id="stateful-bundle-2_last_0" operation_key="stateful-bundle-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="158:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;158:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="13" rc-code="0" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="7557a78dd63ce4c65d8c11dedd989a74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="stateful-bundle-2_monitor_30000" operation_key="stateful-bundle-2_monitor_30000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="123:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;123:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="14" rc-code="0" op-status="0" interval="30000" last-rc-change="1592409571" exec-time="0" queue-time="0" op-digest="712ee29a6c182562060ccfbab6326030"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-1" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-1">
+ <lrm_rsc_op id="stateful-bundle-1_last_0" operation_key="stateful-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="57:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;57:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="11" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="98c4879a95dbe1b4a991ce576dd25733" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-0_last_0" operation_key="stateful-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="53:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;53:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="122" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="111" queue-time="0" op-digest="3a787e755377f774b51e7be6dd61684d"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-1_last_0" operation_key="stateful-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="56:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;56:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="130" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="104" queue-time="0" op-digest="7652b221fbab60cc0bb8a0edc188cf31"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="52:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;52:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="118" rc-code="7" op-status="0" interval="0" last-rc-change="1592409564" last-run="1592409564" exec-time="69" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="55:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;55:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="126" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="136" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="154:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;154:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="140" rc-code="0" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="86" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_monitor_60000" operation_key="stateful-bundle-ip-192.168.122.133_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="155:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;155:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="142" rc-code="0" op-status="0" interval="60000" last-rc-change="1592409567" exec-time="158" queue-time="0" op-digest="be05af11daf7c113782f16adb340ab8e"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-2_last_0" operation_key="stateful-bundle-docker-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="156:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;156:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="143" rc-code="0" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="919" queue-time="0" op-digest="3d5c11040df7f5f8b4f7d0ab9675cf2e"/>
+ <lrm_rsc_op id="stateful-bundle-docker-2_monitor_60000" operation_key="stateful-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="157:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;157:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="147" rc-code="0" op-status="0" interval="60000" last-rc-change="1592409568" exec-time="214" queue-time="0" op-digest="bcd89d41e579f1f7f4ff83e715ab91f6"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-master-rsc1" name="master-rsc1" value="10"/>
+ <nvpair id="status-4-master-rsc2" name="master-rsc2" value="10"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="3" uname="rhel7-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="32:16:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;32:16:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="40" rc-code="0" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="1" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="20:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;20:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="54" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="10:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;10:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="19" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="46" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="30:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;30:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="21" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="85" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="38:3:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;38:3:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="23" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="24" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="remote-rhel7-2" type="remote" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="remote-rhel7-2_last_0" operation_key="remote-rhel7-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="13:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:7;13:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-3" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1592340505" last-run="1592340505" exec-time="0" queue-time="0" op-digest="f1c162fde9f781a7f852b761a6e079e5" op-force-restart=" server " op-restart-digest="f1c162fde9f781a7f852b761a6e079e5"/>
+ </lrm_resource>
+ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container1_last_0" operation_key="container1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="82:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;82:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="51" rc-code="0" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="1323" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
+ </lrm_resource>
+ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container2_last_0" operation_key="container2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="83:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;83:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="59" rc-code="0" op-status="0" interval="0" last-rc-change="1592342393" last-run="1592342393" exec-time="1508" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
+ </lrm_resource>
+ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
+ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="94:18:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;94:18:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="5" rc-code="0" op-status="0" interval="0" last-rc-change="1592342394" last-run="1592342394" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="lxc2_monitor_30000" operation_key="lxc2_monitor_30000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="96:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;96:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="7" rc-code="0" op-status="0" interval="30000" last-rc-change="1592342403" exec-time="0" queue-time="0" op-digest="02a5bcf940fc8d3239701acb11438d6a"/>
+ </lrm_resource>
+ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
+ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="92:18:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;92:18:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="4" rc-code="0" op-status="0" interval="0" last-rc-change="1592342394" last-run="1592342394" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="lxc1_monitor_30000" operation_key="lxc1_monitor_30000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="93:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;93:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="6" rc-code="0" op-status="0" interval="30000" last-rc-change="1592342403" exec-time="0" queue-time="0" op-digest="02a5bcf940fc8d3239701acb11438d6a"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="19:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;19:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="56" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="72" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-0" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-0">
+ <lrm_rsc_op id="stateful-bundle-0_last_0" operation_key="stateful-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="45:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;45:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="11" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="0" queue-time="0" op-digest="bd3b59fd6cf7aabf0f8a6eed8a0e2a71" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-2" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-2">
+ <lrm_rsc_op id="stateful-bundle-2_last_0" operation_key="stateful-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="51:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;51:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="13" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="7557a78dd63ce4c65d8c11dedd989a74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-1" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-1">
+ <lrm_rsc_op id="stateful-bundle-1_last_0" operation_key="stateful-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="48:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;48:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="12" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="98c4879a95dbe1b4a991ce576dd25733" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-0_last_0" operation_key="stateful-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="44:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;44:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="113" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="160" queue-time="0" op-digest="3a787e755377f774b51e7be6dd61684d"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-1_last_0" operation_key="stateful-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="47:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;47:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="121" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="96" queue-time="0" op-digest="7652b221fbab60cc0bb8a0edc188cf31"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="43:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;43:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="109" rc-code="7" op-status="0" interval="0" last-rc-change="1592409564" last-run="1592409564" exec-time="72" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="46:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;46:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="117" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="199" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="49:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;49:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="125" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="141" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-2_last_0" operation_key="stateful-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="50:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;50:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="130" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="77" queue-time="0" op-digest="3d5c11040df7f5f8b4f7d0ab9675cf2e"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-3-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="5" uname="rhel7-5" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="5">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:0:7:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;13:0:7:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="10" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="24:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;24:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;13:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="18" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="48" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="34:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;34:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="20" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="35:2:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;35:2:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="40" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="remote-rhel7-2" type="remote" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="remote-rhel7-2_last_0" operation_key="remote-rhel7-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="15:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:7;15:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-5" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1592340505" last-run="1592340505" exec-time="0" queue-time="0" op-digest="f1c162fde9f781a7f852b761a6e079e5" op-force-restart=" server " op-restart-digest="f1c162fde9f781a7f852b761a6e079e5"/>
+ </lrm_resource>
+ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="28:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;28:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-5" call-id="35" rc-code="7" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="355" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
+ </lrm_resource>
+ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="29:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;29:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-5" call-id="39" rc-code="7" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="340" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
+ </lrm_resource>
+ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
+ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="27:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;27:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-5" call-id="3" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
+ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="26:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;26:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-5" call-id="2" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="25:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;25:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-5" call-id="44" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="44" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-0" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-0">
+ <lrm_rsc_op id="stateful-bundle-0_last_0" operation_key="stateful-bundle-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="146:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;146:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="11" rc-code="0" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="0" queue-time="0" op-digest="bd3b59fd6cf7aabf0f8a6eed8a0e2a71" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="stateful-bundle-0_monitor_30000" operation_key="stateful-bundle-0_monitor_30000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="109:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;109:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="14" rc-code="0" op-status="0" interval="30000" last-rc-change="1592409570" exec-time="0" queue-time="0" op-digest="039dfa92620d65de0a455ebf46bf1bb6"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-2" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-2">
+ <lrm_rsc_op id="stateful-bundle-2_last_0" operation_key="stateful-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="69:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;69:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="13" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="7557a78dd63ce4c65d8c11dedd989a74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-1" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-1">
+ <lrm_rsc_op id="stateful-bundle-1_last_0" operation_key="stateful-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="66:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;66:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="12" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="98c4879a95dbe1b4a991ce576dd25733" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-0_last_0" operation_key="stateful-bundle-docker-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="144:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;144:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="131" rc-code="0" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="1084" queue-time="0" op-digest="3a787e755377f774b51e7be6dd61684d"/>
+ <lrm_rsc_op id="stateful-bundle-docker-0_monitor_60000" operation_key="stateful-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="145:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;145:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="134" rc-code="0" op-status="0" interval="60000" last-rc-change="1592409567" exec-time="300" queue-time="0" op-digest="de3cb764bdefc60ad83989eb8c57a8d0"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-1_last_0" operation_key="stateful-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="65:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;65:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="119" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="107" queue-time="0" op-digest="7652b221fbab60cc0bb8a0edc188cf31"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="142:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;142:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="124" rc-code="0" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="179" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_monitor_60000" operation_key="stateful-bundle-ip-192.168.122.131_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="143:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;143:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="130" rc-code="0" op-status="0" interval="60000" last-rc-change="1592409566" exec-time="180" queue-time="0" op-digest="fc520faa5d1d11c458a6d451e0e08a0e"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="64:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;64:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="115" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="149" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="67:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;67:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="123" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="138" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-2_last_0" operation_key="stateful-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="68:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;68:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="128" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="106" queue-time="0" op-digest="3d5c11040df7f5f8b4f7d0ab9675cf2e"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="5">
+ <instance_attributes id="status-5">
+ <nvpair id="status-5-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-5-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="1" uname="rhel7-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="16:2:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:0;16:2:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-1" call-id="30" rc-code="0" op-status="0" interval="0" last-rc-change="1592340505" last-run="1592340505" exec-time="0" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-0" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-0">
+ <lrm_rsc_op id="stateful-bundle-0_last_0" operation_key="stateful-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="36:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;36:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="12" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="0" queue-time="0" op-digest="bd3b59fd6cf7aabf0f8a6eed8a0e2a71" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-2" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-2">
+ <lrm_rsc_op id="stateful-bundle-2_last_0" operation_key="stateful-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="42:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;42:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="15" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="7557a78dd63ce4c65d8c11dedd989a74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-1" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-1">
+ <lrm_rsc_op id="stateful-bundle-1_last_0" operation_key="stateful-bundle-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="152:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;152:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="14" rc-code="0" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="98c4879a95dbe1b4a991ce576dd25733" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="stateful-bundle-1_monitor_30000" operation_key="stateful-bundle-1_monitor_30000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="116:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;116:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="16" rc-code="0" op-status="0" interval="30000" last-rc-change="1592409570" exec-time="0" queue-time="0" op-digest="3ad2e1d8539d1e6893d4a32eaae0c4de"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-0_last_0" operation_key="stateful-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="35:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;35:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="125" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="76" queue-time="0" op-digest="3a787e755377f774b51e7be6dd61684d"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-1_last_0" operation_key="stateful-bundle-docker-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="150:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;150:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="145" rc-code="0" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="1065" queue-time="0" op-digest="7652b221fbab60cc0bb8a0edc188cf31"/>
+ <lrm_rsc_op id="stateful-bundle-docker-1_monitor_60000" operation_key="stateful-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="151:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;151:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="149" rc-code="0" op-status="0" interval="60000" last-rc-change="1592409568" exec-time="377" queue-time="3" op-digest="fcd9e8b06389130ce68ea7bebcc14800"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="34:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;34:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="121" rc-code="7" op-status="0" interval="0" last-rc-change="1592409564" last-run="1592409564" exec-time="80" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="148:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;148:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="142" rc-code="0" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="108" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_monitor_60000" operation_key="stateful-bundle-ip-192.168.122.132_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="149:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;149:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="144" rc-code="0" op-status="0" interval="60000" last-rc-change="1592409567" exec-time="157" queue-time="0" op-digest="d35d9d188df006e76b4a9cc497a32049"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="40:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;40:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="137" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="66" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-2_last_0" operation_key="stateful-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="41:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;41:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="141" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="64" queue-time="0" op-digest="3d5c11040df7f5f8b4f7d0ab9675cf2e"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="26:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;26:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="20" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="88" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="16:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;16:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="45" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="36:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;36:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="24" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="38:2:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;38:2:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="26" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="31" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="remote-rhel7-2" type="remote" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="remote-rhel7-2_last_0" operation_key="remote-rhel7-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="58:3:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:0;58:3:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-1" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1592340505" last-run="1592340505" exec-time="0" queue-time="0" op-digest="f1c162fde9f781a7f852b761a6e079e5" op-force-restart=" server " op-restart-digest="f1c162fde9f781a7f852b761a6e079e5"/>
+ <lrm_rsc_op id="remote-rhel7-2_monitor_60000" operation_key="remote-rhel7-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="57:4:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:0;57:4:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-1" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1592340508" exec-time="0" queue-time="0" op-digest="fd28010af096484e2129329ce206b589"/>
+ </lrm_resource>
+ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="16:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;16:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="61" rc-code="7" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="178" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
+ </lrm_resource>
+ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="15:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;15:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="65" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="210" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
+ </lrm_resource>
+ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
+ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="18:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;18:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592342393" last-run="1592342393" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
+ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="17:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;17:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="4" rc-code="7" op-status="0" interval="0" last-rc-change="1592342393" last-run="1592342393" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="16:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;16:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="70" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="17" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-1-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="remote-rhel7-2" uname="remote-rhel7-2" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
+ <lrm id="remote-rhel7-2">
+ <lrm_resources>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="25:4:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:0;25:4:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-1" call-id="13" rc-code="0" op-status="0" interval="0" last-rc-change="1592340508" last-run="1592340508" exec-time="187" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="26:5:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:0;26:5:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-1" call-id="29" rc-code="0" op-status="0" interval="11000" last-rc-change="1592340509" exec-time="16" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="45:1:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" transition-magic="0:0;45:1:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" exit-reason="" on_node="rhel7-1" call-id="77" rc-code="0" op-status="0" interval="0" last-rc-change="1592340727" last-run="1592340727" exec-time="184" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_10000" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="47:2:8:00a37a7a-7ac2-4d2b-b272-e4526634552b" transition-magic="0:8;47:2:8:00a37a7a-7ac2-4d2b-b272-e4526634552b" exit-reason="" on_node="rhel7-1" call-id="85" rc-code="8" op-status="0" interval="10000" last-rc-change="1592340727" exec-time="20" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="rsc2_last_failure_0" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="47:2:8:00a37a7a-7ac2-4d2b-b272-e4526634552b" transition-magic="0:7;47:2:8:00a37a7a-7ac2-4d2b-b272-e4526634552b" exit-reason="" on_node="rhel7-1" call-id="85" rc-code="7" op-status="0" interval="10000" last-rc-change="1592340757" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="14:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;14:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="49" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="156" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
+ </lrm_resource>
+ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;13:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="53" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="113" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="14:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;14:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="58" rc-code="7" op-status="0" interval="0" last-rc-change="1592342393" last-run="1592342393" exec-time="13" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="28:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;28:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="341" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="98" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-0_last_0" operation_key="stateful-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="29:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;29:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="345" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="89" queue-time="0" op-digest="3a787e755377f774b51e7be6dd61684d"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="30:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;30:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="349" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="86" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-1_last_0" operation_key="stateful-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="31:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;31:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="353" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="78" queue-time="0" op-digest="7652b221fbab60cc0bb8a0edc188cf31"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="32:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;32:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="357" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="57" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-2_last_0" operation_key="stateful-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="33:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;33:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="361" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="59" queue-time="0" op-digest="3d5c11040df7f5f8b4f7d0ab9675cf2e"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="remote-rhel7-2">
+ <instance_attributes id="status-remote-rhel7-2">
+ <nvpair id="status-remote-rhel7-2-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-remote-rhel7-2-master-rsc2" name="master-rsc2" value="10"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="lxc2" uname="lxc2" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
+ <lrm id="lxc2">
+ <lrm_resources>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="38:19:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;38:19:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592342401" last-run="1592342401" exec-time="187" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="33:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;33:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592342402" exec-time="36" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="59:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;59:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="23" rc-code="0" op-status="0" interval="0" last-rc-change="1592342402" last-run="1592342402" exec-time="192" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="59:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;59:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="29" rc-code="0" op-status="0" interval="11000" last-rc-change="1592342402" exec-time="45" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="80:22:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;80:22:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="42" rc-code="0" op-status="0" interval="0" last-rc-change="1592342404" last-run="1592342404" exec-time="195" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="lxc-ms_monitor_10000" operation_key="lxc-ms_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="83:24:8:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:8;83:24:8:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="62" rc-code="8" op-status="0" interval="10000" last-rc-change="1592342844" exec-time="33" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="lxc-ms_last_failure_0" operation_key="lxc-ms_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="83:24:8:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;83:24:8:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="62" rc-code="7" op-status="0" interval="10000" last-rc-change="1592342864" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="25:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;25:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="87" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="85" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="26:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;26:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="91" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="53" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="27:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;27:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="95" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="96" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="lxc2">
+ <instance_attributes id="status-lxc2">
+ <nvpair id="status-lxc2-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-lxc2-master-rsc2" name="master-rsc2" value="5"/>
+ <nvpair id="status-lxc2-master-lxc-ms" name="master-lxc-ms" value="10"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="lxc1" uname="lxc1" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
+ <lrm id="lxc1">
+ <lrm_resources>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="36:19:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;36:19:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="12" rc-code="0" op-status="0" interval="0" last-rc-change="1592342401" last-run="1592342401" exec-time="236" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="36:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;36:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="18" rc-code="0" op-status="0" interval="11000" last-rc-change="1592342402" exec-time="52" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="57:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;57:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="19" rc-code="0" op-status="0" interval="0" last-rc-change="1592342402" last-run="1592342402" exec-time="218" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="62:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;62:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="25" rc-code="0" op-status="0" interval="11000" last-rc-change="1592342402" exec-time="67" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="79:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;79:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="36" rc-code="0" op-status="0" interval="0" last-rc-change="1592342403" last-run="1592342403" exec-time="230" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="lxc-ms_monitor_10000" operation_key="lxc-ms_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="83:22:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;83:22:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="42" rc-code="0" op-status="0" interval="10000" last-rc-change="1592342404" exec-time="47" queue-time="0" op-digest="8f6a313464b7f9e3a31cb448458b700e"/>
+ <lrm_rsc_op id="lxc-ms_monitor_11000" operation_key="lxc-ms_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="86:24:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;86:24:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="57" rc-code="0" op-status="0" interval="11000" last-rc-change="1592342844" exec-time="32" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="22:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;22:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="87" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="66" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="23:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;23:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="91" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="53" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="24:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;24:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="95" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="61" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="lxc1">
+ <instance_attributes id="status-lxc1">
+ <nvpair id="status-lxc1-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-lxc1-master-rsc2" name="master-rsc2" value="5"/>
+ <nvpair id="status-lxc1-master-lxc-ms" name="master-lxc-ms" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="stateful-bundle-2" uname="stateful-bundle-2" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0">
+ <lrm id="stateful-bundle-2">
+ <lrm_resources>
+ <lrm_resource id="bundled" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="bundled_last_0" operation_key="bundled_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="132:5:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;132:5:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="8" rc-code="0" op-status="0" interval="0" last-rc-change="1592409572" last-run="1592409572" exec-time="241" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="bundled_monitor_11000" operation_key="bundled_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="134:6:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;134:6:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="14" rc-code="0" op-status="0" interval="11000" last-rc-change="1592409572" exec-time="16" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="stateful-bundle-2">
+ <instance_attributes id="status-stateful-bundle-2">
+ <nvpair id="status-stateful-bundle-2-master-bundled" name="master-bundled" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="stateful-bundle-0" uname="stateful-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
+ <lrm id="stateful-bundle-0">
+ <lrm_resources>
+ <lrm_resource id="bundled" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="bundled_last_0" operation_key="bundled_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="128:6:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;128:6:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="14" rc-code="0" op-status="0" interval="0" last-rc-change="1592409572" last-run="1592409572" exec-time="164" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="bundled_monitor_10000" operation_key="bundled_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="131:7:8:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:8;131:7:8:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="20" rc-code="8" op-status="0" interval="10000" last-rc-change="1592409572" exec-time="34" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="bundled_last_failure_0" operation_key="bundled_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="131:7:8:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;131:7:8:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="20" rc-code="7" op-status="0" interval="10000" last-rc-change="1592409773" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="stateful-bundle-0">
+ <instance_attributes id="status-stateful-bundle-0">
+ <nvpair id="status-stateful-bundle-0-master-bundled" name="master-bundled" value="10"/>
+ <nvpair id="status-stateful-bundle-0-fail-count-bundled.monitor_10000" name="fail-count-bundled#monitor_10000" value="1"/>
+ <nvpair id="status-stateful-bundle-0-last-failure-bundled.monitor_10000" name="last-failure-bundled#monitor_10000" value="1592409773"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="stateful-bundle-1" uname="stateful-bundle-1" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0">
+ <lrm id="stateful-bundle-1">
+ <lrm_resources>
+ <lrm_resource id="bundled" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="bundled_last_0" operation_key="bundled_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="129:4:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;129:4:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="8" rc-code="0" op-status="0" interval="0" last-rc-change="1592409571" last-run="1592409571" exec-time="225" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="bundled_monitor_11000" operation_key="bundled_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="131:5:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;131:5:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="14" rc-code="0" op-status="0" interval="11000" last-rc-change="1592409572" exec-time="14" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="stateful-bundle-1">
+ <instance_attributes id="status-stateful-bundle-1">
+ <nvpair id="status-stateful-bundle-1-master-bundled" name="master-bundled" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/cts/scheduler/on_fail_demote2.dot b/cts/scheduler/on_fail_demote2.dot
new file mode 100644
index 0000000..06193cb
--- /dev/null
+++ b/cts/scheduler/on_fail_demote2.dot
@@ -0,0 +1,22 @@
+ digraph "g" {
+"Cancel rsc1_monitor_10000 rhel7-4" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"Cancel rsc1_monitor_10000 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"Cancel rsc1_monitor_11000 rhel7-3" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"Cancel rsc1_monitor_11000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_demoted_0" -> "rsc1-clone_promote_0" [ style = bold]
+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promote_0" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"rsc1-clone_promote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1_demote_0 rhel7-4" -> "rsc1_monitor_11000 rhel7-4" [ style = bold]
+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc1_monitor_10000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc1_monitor_11000 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc1_promote_0 rhel7-3" -> "rsc1-clone_promoted_0" [ style = bold]
+"rsc1_promote_0 rhel7-3" -> "rsc1_monitor_10000 rhel7-3" [ style = bold]
+"rsc1_promote_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/cts/scheduler/on_fail_demote2.exp b/cts/scheduler/on_fail_demote2.exp
new file mode 100644
index 0000000..492e86f
--- /dev/null
+++ b/cts/scheduler/on_fail_demote2.exp
@@ -0,0 +1,125 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="1" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="17" operation="monitor" operation_key="rsc1_monitor_11000" internal_operation_key="rsc1:0_monitor_11000" on_node="rhel7-4" on_node_uuid="4">
+ <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="11000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Slave" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="2" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="2" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-4" on_node_uuid="4">
+ <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="1" operation="cancel" operation_key="rsc1_monitor_10000" internal_operation_key="rsc1:0_monitor_10000" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="34" operation="demote" operation_key="rsc1-clone_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="1" operation="cancel" operation_key="rsc1_monitor_10000" internal_operation_key="rsc1:0_monitor_10000" on_node="rhel7-4" on_node_uuid="4">
+ <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_fail="demote" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_operation="monitor" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Master" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="21" operation="monitor" operation_key="rsc1_monitor_10000" internal_operation_key="rsc1:1_monitor_10000" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="rsc1" long-id="rsc1:1" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_fail="demote" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="8" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Master" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="20" operation="promote" operation_key="rsc1_promote_0" internal_operation_key="rsc1:1_promote_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <rsc_op id="20" operation="promote" operation_key="rsc1_promote_0" internal_operation_key="rsc1:1_promote_0" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="rsc1" long-id="rsc1:1" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="4" operation="cancel" operation_key="rsc1_monitor_11000" internal_operation_key="rsc1:1_monitor_11000" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="32" operation="promote" operation_key="rsc1-clone_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <rsc_op id="4" operation="cancel" operation_key="rsc1_monitor_11000" internal_operation_key="rsc1:1_monitor_11000" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="rsc1" long-id="rsc1:1" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="11000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_operation="monitor" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Slave" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="6" priority="1000000">
+ <action_set>
+ <pseudo_event id="35" operation="demoted" operation_key="rsc1-clone_demoted_0">
+ <attributes CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="2" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="34" operation="demote" operation_key="rsc1-clone_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <pseudo_event id="34" operation="demote" operation_key="rsc1-clone_demote_0">
+ <attributes CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="8" priority="1000000">
+ <action_set>
+ <pseudo_event id="33" operation="promoted" operation_key="rsc1-clone_promoted_0">
+ <attributes CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="20" operation="promote" operation_key="rsc1_promote_0" internal_operation_key="rsc1:1_promote_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <pseudo_event id="32" operation="promote" operation_key="rsc1-clone_promote_0">
+ <attributes CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="35" operation="demoted" operation_key="rsc1-clone_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/cts/scheduler/on_fail_demote2.scores b/cts/scheduler/on_fail_demote2.scores
new file mode 100644
index 0000000..25aea90
--- /dev/null
+++ b/cts/scheduler/on_fail_demote2.scores
@@ -0,0 +1,127 @@
+Allocation scores:
+Using the original execution date of: 2020-06-16 19:23:21Z
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-2: 6
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-2: 6
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-2: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-2: 6
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-2: 6
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY
+rsc1:0 promotion score on rhel7-4: -INFINITY
+rsc1:1 promotion score on rhel7-3: 5
+rsc1:2 promotion score on rhel7-5: 5
+rsc1:3 promotion score on rhel7-1: 5
+rsc1:4 promotion score on rhel7-2: 5
+rsc2:0 promotion score on rhel7-4: 10
+rsc2:1 promotion score on rhel7-3: 5
+rsc2:2 promotion score on rhel7-5: 5
+rsc2:3 promotion score on rhel7-1: 5
+rsc2:4 promotion score on rhel7-2: 5
diff --git a/cts/scheduler/on_fail_demote2.summary b/cts/scheduler/on_fail_demote2.summary
new file mode 100644
index 0000000..795a11d
--- /dev/null
+++ b/cts/scheduler/on_fail_demote2.summary
@@ -0,0 +1,41 @@
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Current cluster status:
+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-1
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ rsc1 (ocf::pacemaker:Stateful): FAILED Master rhel7-4
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ rhel7-4 ]
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+
+Transition Summary:
+ * Demote rsc1:0 ( Master -> Slave rhel7-4 )
+ * Promote rsc1:1 ( Slave -> Master rhel7-3 )
+
+Executing cluster transition:
+ * Resource action: rsc1 cancel=10000 on rhel7-4
+ * Resource action: rsc1 cancel=11000 on rhel7-3
+ * Pseudo action: rsc1-clone_demote_0
+ * Resource action: rsc1 demote on rhel7-4
+ * Pseudo action: rsc1-clone_demoted_0
+ * Pseudo action: rsc1-clone_promote_0
+ * Resource action: rsc1 monitor=11000 on rhel7-4
+ * Resource action: rsc1 promote on rhel7-3
+ * Pseudo action: rsc1-clone_promoted_0
+ * Resource action: rsc1 monitor=10000 on rhel7-3
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Revised cluster status:
+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-1
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Masters: [ rhel7-3 ]
+ Slaves: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ rhel7-4 ]
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+
diff --git a/cts/scheduler/on_fail_demote2.xml b/cts/scheduler/on_fail_demote2.xml
new file mode 100644
index 0000000..ae91633
--- /dev/null
+++ b/cts/scheduler/on_fail_demote2.xml
@@ -0,0 +1,221 @@
+<cib crm_feature_set="3.4.0" validate-with="pacemaker-3.4" epoch="62" num_updates="98" admin_epoch="1" cib-last-written="Tue Jun 16 14:21:19 2020" update-origin="rhel7-2" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2" execution-date="1592335401">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/>
+ <nvpair id="cts-start-failure-is-fatal" name="start-failure-is-fatal" value="false"/>
+ <nvpair id="cts-pe-input-series-max" name="pe-input-series-max" value="5000"/>
+ <nvpair id="cts-shutdown-escalation" name="shutdown-escalation" value="5min"/>
+ <nvpair id="cts-batch-limit" name="batch-limit" value="10"/>
+ <nvpair id="cts-dc-deadtime" name="dc-deadtime" value="5s"/>
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-578.6e1b582.git.el7_8-6e1b582"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="mycluster"/>
+ <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1591654576"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="rhel7-1">
+ <instance_attributes id="rhel7-1-1">
+ <nvpair id="rhel7-1-1-cts-fencing" name="cts-fencing" value="levels-and"/>
+ </instance_attributes>
+ </node>
+ <node id="2" uname="rhel7-2">
+ <instance_attributes id="rhel7-2-1">
+ <nvpair id="rhel7-2-1-cts-fencing" name="cts-fencing" value="levels-and"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="rhel7-3"/>
+ <node id="4" uname="rhel7-4">
+ <instance_attributes id="nodes-4">
+ <nvpair id="nodes-4-standby" name="standby" value="off"/>
+ </instance_attributes>
+ </node>
+ <node id="5" uname="rhel7-5"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="Fencing" type="fence_xvm">
+ <meta_attributes id="Fencing-meta"/>
+ <instance_attributes id="Fencing-params">
+ <nvpair id="Fencing-key_file" name="key_file" value="/etc/pacemaker/fence_xvm.key"/>
+ <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/>
+ <nvpair id="Fencing-pcmk_host_map" name="pcmk_host_map" value="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
+ <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 rhel7-3 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5"/>
+ </instance_attributes>
+ <operations>
+ <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/>
+ <op id="Fencing-stop-0" interval="0" name="stop" timeout="60s"/>
+ <op id="Fencing-start-0" interval="0" name="start" timeout="60s"/>
+ </operations>
+ </primitive>
+ <clone id="rsc1-clone">
+ <meta_attributes id="rsc1-meta_attributes">
+ <nvpair id="rsc1-meta_attributes-promotable" name="promotable" value="true"/>
+ </meta_attributes>
+ <primitive class="ocf" id="rsc1" provider="pacemaker" type="Stateful">
+ <operations>
+ <op id="rsc1-demote-interval-0s" interval="0s" name="demote" timeout="10s"/>
+ <op id="rsc1-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="rsc1-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ <op id="rsc1-notify-interval-0s" interval="0s" name="notify" timeout="5s"/>
+ <op id="rsc1-promote-interval-0s" interval="0s" name="promote" timeout="10s"/>
+ <op id="rsc1-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+ <op id="rsc1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+ </operations>
+ </primitive>
+ </clone>
+ <master id="rsc2-master">
+ <primitive class="ocf" id="rsc2" provider="pacemaker" type="Stateful">
+ <operations>
+ <op id="rsc2-demote-interval-0s" interval="0s" name="demote" timeout="10s"/>
+ <op id="rsc2-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="rsc2-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ <op id="rsc2-notify-interval-0s" interval="0s" name="notify" timeout="5s"/>
+ <op id="rsc2-promote-interval-0s" interval="0s" name="promote" timeout="10s"/>
+ <op id="rsc2-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+ <op id="rsc2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+ </operations>
+ </primitive>
+ </master>
+ </resources>
+ <constraints>
+ <rsc_location id="location-rsc1-clone" rsc="rsc1-clone">
+ <rule id="location-rsc1-clone-rule" role="master" score="-INFINITY">
+ <expression attribute="fail-count-rsc1#monitor_10000" id="location-rsc1-clone-rule-expr" operation="gt" value="0"/>
+ </rule>
+ </rsc_location>
+ </constraints>
+ <op_defaults>
+ <meta_attributes id="cts-op_defaults-meta">
+ <nvpair id="cts-op_defaults-timeout" name="timeout" value="90s"/>
+ </meta_attributes>
+ </op_defaults>
+ <alerts>
+ <alert id="alert-1" path="/var/lib/pacemaker/notify.sh">
+ <recipient id="alert-1-recipient-1" value="/run/crm/alert.log"/>
+ </alert>
+ </alerts>
+ </configuration>
+ <status>
+ <node_state id="4" uname="rhel7-4" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="10:0:7:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;10:0:7:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="6" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="6:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;6:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="18" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="94" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_10000" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="12:2:8:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:8;12:2:8:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="20" rc-code="8" op-status="0" interval="10000" last-rc-change="1592335281" exec-time="26" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="rsc1_last_failure_0" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="12:2:8:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;12:2:8:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="20" rc-code="7" op-status="0" interval="10000" last-rc-change="1592335401" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="34:3:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;34:3:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="24" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="42" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_10000" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="37:4:8:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:8;37:4:8:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="26" rc-code="8" op-status="0" interval="10000" last-rc-change="1592335281" exec-time="18" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-master-rsc1" name="master-rsc1" value="10"/>
+ <nvpair id="status-4-master-rsc2" name="master-rsc2" value="10"/>
+ <nvpair id="status-4-fail-count-rsc1.monitor_10000" name="fail-count-rsc1#monitor_10000" value="1"/>
+ <nvpair id="status-4-last-failure-rsc1.monitor_10000" name="last-failure-rsc1#monitor_10000" value="1592335401"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="3" uname="rhel7-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="7:0:7:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;7:0:7:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="6" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="20:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;20:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="54" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="10:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;10:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="19" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="46" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="30:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;30:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="21" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="85" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="38:3:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;38:3:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="23" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="24" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-3-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="5" uname="rhel7-5" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="5">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:0:7:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;13:0:7:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="10" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="24:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;24:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;13:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="18" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="48" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="34:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;34:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="20" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="35:2:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;35:2:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="40" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="5">
+ <instance_attributes id="status-5">
+ <nvpair id="status-5-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-5-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="1" uname="rhel7-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="16:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;16:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="6" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="98" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="17:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;17:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="8" rc-code="0" op-status="0" interval="120000" last-rc-change="1592335280" exec-time="35" queue-time="0" op-digest="cb34bc19df153021ce8f301baa293f35"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="26:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;26:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="20" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="88" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="16:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;16:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="45" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="36:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;36:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="24" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="38:2:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;38:2:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="26" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="31" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-1-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="rhel7-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="4:0:7:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;4:0:7:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-2" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="10" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="18:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;18:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-2" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="57" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="19:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;19:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-2" call-id="18" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="30" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="28:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;28:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-2" call-id="20" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="64" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="41:2:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;41:2:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-2" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="36" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-2-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/cts/scheduler/on_fail_demote3.dot b/cts/scheduler/on_fail_demote3.dot
new file mode 100644
index 0000000..e78325b
--- /dev/null
+++ b/cts/scheduler/on_fail_demote3.dot
@@ -0,0 +1,12 @@
+ digraph "g" {
+"Cancel rsc1_monitor_10000 rhel7-4" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"Cancel rsc1_monitor_10000 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1_demote_0 rhel7-4" -> "rsc1_monitor_11000 rhel7-4" [ style = bold]
+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc1_monitor_11000 rhel7-4" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/cts/scheduler/on_fail_demote3.exp b/cts/scheduler/on_fail_demote3.exp
new file mode 100644
index 0000000..ed6bd6d
--- /dev/null
+++ b/cts/scheduler/on_fail_demote3.exp
@@ -0,0 +1,63 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="1" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="17" operation="monitor" operation_key="rsc1_monitor_11000" internal_operation_key="rsc1:0_monitor_11000" on_node="rhel7-4" on_node_uuid="4">
+ <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="11000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Slave" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="2" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="2" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-4" on_node_uuid="4">
+ <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="1" operation="cancel" operation_key="rsc1_monitor_10000" internal_operation_key="rsc1:0_monitor_10000" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="32" operation="demote" operation_key="rsc1-clone_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="1" operation="cancel" operation_key="rsc1_monitor_10000" internal_operation_key="rsc1:0_monitor_10000" on_node="rhel7-4" on_node_uuid="4">
+ <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_fail="demote" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_operation="monitor" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Master" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="3" priority="1000000">
+ <action_set>
+ <pseudo_event id="33" operation="demoted" operation_key="rsc1-clone_demoted_0">
+ <attributes CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="2" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="32" operation="demote" operation_key="rsc1-clone_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <pseudo_event id="32" operation="demote" operation_key="rsc1-clone_demote_0">
+ <attributes CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+</transition_graph>
diff --git a/cts/scheduler/on_fail_demote3.scores b/cts/scheduler/on_fail_demote3.scores
new file mode 100644
index 0000000..a85639a
--- /dev/null
+++ b/cts/scheduler/on_fail_demote3.scores
@@ -0,0 +1,127 @@
+Allocation scores:
+Using the original execution date of: 2020-06-16 19:23:21Z
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-2: 6
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-2: 6
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-2: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-2: 6
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-2: 6
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY
+rsc1:0 promotion score on rhel7-4: -INFINITY
+rsc1:1 promotion score on rhel7-3: -INFINITY
+rsc1:2 promotion score on rhel7-5: -INFINITY
+rsc1:3 promotion score on rhel7-1: -INFINITY
+rsc1:4 promotion score on rhel7-2: -INFINITY
+rsc2:0 promotion score on rhel7-4: 10
+rsc2:1 promotion score on rhel7-3: 5
+rsc2:2 promotion score on rhel7-5: 5
+rsc2:3 promotion score on rhel7-1: 5
+rsc2:4 promotion score on rhel7-2: 5
diff --git a/cts/scheduler/on_fail_demote3.summary b/cts/scheduler/on_fail_demote3.summary
new file mode 100644
index 0000000..f1173fd
--- /dev/null
+++ b/cts/scheduler/on_fail_demote3.summary
@@ -0,0 +1,34 @@
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Current cluster status:
+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-1
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ rsc1 (ocf::pacemaker:Stateful): FAILED Master rhel7-4
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ rhel7-4 ]
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+
+Transition Summary:
+ * Demote rsc1:0 ( Master -> Slave rhel7-4 )
+
+Executing cluster transition:
+ * Resource action: rsc1 cancel=10000 on rhel7-4
+ * Pseudo action: rsc1-clone_demote_0
+ * Resource action: rsc1 demote on rhel7-4
+ * Pseudo action: rsc1-clone_demoted_0
+ * Resource action: rsc1 monitor=11000 on rhel7-4
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Revised cluster status:
+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-1
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ rhel7-4 ]
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+
diff --git a/cts/scheduler/on_fail_demote3.xml b/cts/scheduler/on_fail_demote3.xml
new file mode 100644
index 0000000..a7b6806
--- /dev/null
+++ b/cts/scheduler/on_fail_demote3.xml
@@ -0,0 +1,221 @@
+<cib crm_feature_set="3.4.0" validate-with="pacemaker-3.4" epoch="62" num_updates="98" admin_epoch="1" cib-last-written="Tue Jun 16 14:21:19 2020" update-origin="rhel7-2" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2" execution-date="1592335401">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/>
+ <nvpair id="cts-start-failure-is-fatal" name="start-failure-is-fatal" value="false"/>
+ <nvpair id="cts-pe-input-series-max" name="pe-input-series-max" value="5000"/>
+ <nvpair id="cts-shutdown-escalation" name="shutdown-escalation" value="5min"/>
+ <nvpair id="cts-batch-limit" name="batch-limit" value="10"/>
+ <nvpair id="cts-dc-deadtime" name="dc-deadtime" value="5s"/>
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-578.6e1b582.git.el7_8-6e1b582"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="mycluster"/>
+ <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1591654576"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="rhel7-1">
+ <instance_attributes id="rhel7-1-1">
+ <nvpair id="rhel7-1-1-cts-fencing" name="cts-fencing" value="levels-and"/>
+ </instance_attributes>
+ </node>
+ <node id="2" uname="rhel7-2">
+ <instance_attributes id="rhel7-2-1">
+ <nvpair id="rhel7-2-1-cts-fencing" name="cts-fencing" value="levels-and"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="rhel7-3"/>
+ <node id="4" uname="rhel7-4">
+ <instance_attributes id="nodes-4">
+ <nvpair id="nodes-4-standby" name="standby" value="off"/>
+ </instance_attributes>
+ </node>
+ <node id="5" uname="rhel7-5"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="Fencing" type="fence_xvm">
+ <meta_attributes id="Fencing-meta"/>
+ <instance_attributes id="Fencing-params">
+ <nvpair id="Fencing-key_file" name="key_file" value="/etc/pacemaker/fence_xvm.key"/>
+ <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/>
+ <nvpair id="Fencing-pcmk_host_map" name="pcmk_host_map" value="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
+ <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 rhel7-3 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5"/>
+ </instance_attributes>
+ <operations>
+ <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/>
+ <op id="Fencing-stop-0" interval="0" name="stop" timeout="60s"/>
+ <op id="Fencing-start-0" interval="0" name="start" timeout="60s"/>
+ </operations>
+ </primitive>
+ <clone id="rsc1-clone">
+ <meta_attributes id="rsc1-meta_attributes">
+ <nvpair id="rsc1-meta_attributes-promotable" name="promotable" value="true"/>
+ </meta_attributes>
+ <primitive class="ocf" id="rsc1" provider="pacemaker" type="Stateful">
+ <operations>
+ <op id="rsc1-demote-interval-0s" interval="0s" name="demote" timeout="10s"/>
+ <op id="rsc1-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="rsc1-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ <op id="rsc1-notify-interval-0s" interval="0s" name="notify" timeout="5s"/>
+ <op id="rsc1-promote-interval-0s" interval="0s" name="promote" timeout="10s"/>
+ <op id="rsc1-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+ <op id="rsc1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+ </operations>
+ </primitive>
+ </clone>
+ <master id="rsc2-master">
+ <primitive class="ocf" id="rsc2" provider="pacemaker" type="Stateful">
+ <operations>
+ <op id="rsc2-demote-interval-0s" interval="0s" name="demote" timeout="10s"/>
+ <op id="rsc2-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="rsc2-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ <op id="rsc2-notify-interval-0s" interval="0s" name="notify" timeout="5s"/>
+ <op id="rsc2-promote-interval-0s" interval="0s" name="promote" timeout="10s"/>
+ <op id="rsc2-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+ <op id="rsc2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+ </operations>
+ </primitive>
+ </master>
+ </resources>
+ <constraints>
+ <rsc_location id="location-rsc1-clone" rsc="rsc1-clone">
+ <rule id="location-rsc1-clone-rule" role="master" score="-INFINITY">
+ <date_expression id="date_expr1" operation="gt" start="2020-01-01"/>
+ </rule>
+ </rsc_location>
+ </constraints>
+ <op_defaults>
+ <meta_attributes id="cts-op_defaults-meta">
+ <nvpair id="cts-op_defaults-timeout" name="timeout" value="90s"/>
+ </meta_attributes>
+ </op_defaults>
+ <alerts>
+ <alert id="alert-1" path="/var/lib/pacemaker/notify.sh">
+ <recipient id="alert-1-recipient-1" value="/run/crm/alert.log"/>
+ </alert>
+ </alerts>
+ </configuration>
+ <status>
+ <node_state id="4" uname="rhel7-4" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="10:0:7:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;10:0:7:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="6" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="6:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;6:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="18" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="94" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_10000" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="12:2:8:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:8;12:2:8:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="20" rc-code="8" op-status="0" interval="10000" last-rc-change="1592335281" exec-time="26" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="rsc1_last_failure_0" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="12:2:8:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;12:2:8:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="20" rc-code="7" op-status="0" interval="10000" last-rc-change="1592335401" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="34:3:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;34:3:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="24" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="42" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_10000" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="37:4:8:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:8;37:4:8:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="26" rc-code="8" op-status="0" interval="10000" last-rc-change="1592335281" exec-time="18" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-master-rsc1" name="master-rsc1" value="10"/>
+ <nvpair id="status-4-master-rsc2" name="master-rsc2" value="10"/>
+ <nvpair id="status-4-fail-count-rsc1.monitor_10000" name="fail-count-rsc1#monitor_10000" value="1"/>
+ <nvpair id="status-4-last-failure-rsc1.monitor_10000" name="last-failure-rsc1#monitor_10000" value="1592335401"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="3" uname="rhel7-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="7:0:7:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;7:0:7:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="6" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="20:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;20:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="54" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="10:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;10:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="19" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="46" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="30:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;30:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="21" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="85" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="38:3:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;38:3:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="23" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="24" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-3-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="5" uname="rhel7-5" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="5">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:0:7:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;13:0:7:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="10" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="24:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;24:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;13:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="18" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="48" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="34:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;34:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="20" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="35:2:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;35:2:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="40" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="5">
+ <instance_attributes id="status-5">
+ <nvpair id="status-5-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-5-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="1" uname="rhel7-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="16:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;16:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="6" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="98" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="17:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;17:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="8" rc-code="0" op-status="0" interval="120000" last-rc-change="1592335280" exec-time="35" queue-time="0" op-digest="cb34bc19df153021ce8f301baa293f35"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="26:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;26:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="20" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="88" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="16:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;16:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="45" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="36:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;36:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="24" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="38:2:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;38:2:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="26" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="31" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-1-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="rhel7-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="4:0:7:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;4:0:7:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-2" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="10" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="18:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;18:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-2" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="57" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="19:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;19:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-2" call-id="18" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="30" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="28:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;28:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-2" call-id="20" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="64" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="41:2:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;41:2:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-2" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="36" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-2-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
diff --git a/cts/scheduler/on_fail_demote4.dot b/cts/scheduler/on_fail_demote4.dot
new file mode 100644
index 0000000..4715cd3
--- /dev/null
+++ b/cts/scheduler/on_fail_demote4.dot
@@ -0,0 +1,383 @@
+ digraph "g" {
+"Cancel rsc1_monitor_11000 rhel7-3" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"Cancel rsc1_monitor_11000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"Cancel rsc2_monitor_11000 rhel7-3" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"Cancel rsc2_monitor_11000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"Fencing_monitor_120000 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"Fencing_start_0 rhel7-5" -> "Fencing_monitor_120000 rhel7-5" [ style = bold]
+"Fencing_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"Fencing_stop_0 rhel7-4" -> "Fencing_start_0 rhel7-5" [ style = bold]
+"Fencing_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"bundled_demote_0 stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"bundled_demote_0 stateful-bundle-0" -> "bundled_stop_0 stateful-bundle-0" [ style = bold]
+"bundled_demote_0 stateful-bundle-0" -> "stateful-bundle-master_demoted_0" [ style = bold]
+"bundled_demote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="orange"]
+"bundled_monitor_10000 stateful-bundle-0" [ style=bold color="green" fontcolor="black"]
+"bundled_monitor_11000 stateful-bundle-2" [ style=bold color="green" fontcolor="black"]
+"bundled_promote_0 stateful-bundle-0" -> "bundled_monitor_10000 stateful-bundle-0" [ style = bold]
+"bundled_promote_0 stateful-bundle-0" -> "stateful-bundle-master_promoted_0" [ style = bold]
+"bundled_promote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"]
+"bundled_start_0 stateful-bundle-0" -> "bundled_monitor_10000 stateful-bundle-0" [ style = bold]
+"bundled_start_0 stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"bundled_start_0 stateful-bundle-0" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"bundled_start_0 stateful-bundle-0" -> "stateful-bundle-master_running_0" [ style = bold]
+"bundled_start_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"]
+"bundled_start_0 stateful-bundle-2" -> "bundled_monitor_11000 stateful-bundle-2" [ style = bold]
+"bundled_start_0 stateful-bundle-2" -> "stateful-bundle-master_running_0" [ style = bold]
+"bundled_start_0 stateful-bundle-2" [ style=bold color="green" fontcolor="black"]
+"bundled_stop_0 stateful-bundle-0" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"bundled_stop_0 stateful-bundle-0" -> "stateful-bundle-master_stopped_0" [ style = bold]
+"bundled_stop_0 stateful-bundle-0" [ style=bold color="green" fontcolor="orange"]
+"bundled_stop_0 stateful-bundle-2" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"bundled_stop_0 stateful-bundle-2" -> "bundled_stop_0 stateful-bundle-0" [ style = bold]
+"bundled_stop_0 stateful-bundle-2" -> "stateful-bundle-master_stopped_0" [ style = bold]
+"bundled_stop_0 stateful-bundle-2" [ style=bold color="green" fontcolor="orange"]
+"container2_monitor_20000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"container2_start_0 rhel7-3" -> "container2_monitor_20000 rhel7-3" [ style = bold]
+"container2_start_0 rhel7-3" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"container2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"container2_start_0 rhel7-3" -> "lxc2_start_0 rhel7-3" [ style = bold]
+"container2_start_0 rhel7-3" -> "rsc1_start_0 lxc2" [ style = bold]
+"container2_start_0 rhel7-3" -> "rsc2_start_0 lxc2" [ style = bold]
+"container2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"container2_stop_0 rhel7-3" -> "container2_start_0 rhel7-3" [ style = bold]
+"container2_stop_0 rhel7-3" -> "stonith 'reboot' lxc2" [ style = bold]
+"container2_stop_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold]
+"lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc2" [ style = bold]
+"lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold]
+"lxc-ms-master_demoted_0" -> "lxc-ms-master_start_0" [ style = bold]
+"lxc-ms-master_demoted_0" -> "lxc-ms-master_stop_0" [ style = bold]
+"lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_running_0" -> "lxc-ms-master_promote_0" [ style = bold]
+"lxc-ms-master_running_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_start_0" -> "lxc-ms-master_running_0" [ style = bold]
+"lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"lxc-ms-master_start_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_stop_0" -> "lxc-ms-master_stopped_0" [ style = bold]
+"lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc2" [ style = bold]
+"lxc-ms-master_stop_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_stopped_0" -> "lxc-ms-master_promote_0" [ style = bold]
+"lxc-ms-master_stopped_0" -> "lxc-ms-master_start_0" [ style = bold]
+"lxc-ms-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms_demote_0 lxc2" -> "lxc-ms-master_demoted_0" [ style = bold]
+"lxc-ms_demote_0 lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc-ms_demote_0 lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold]
+"lxc-ms_demote_0 lxc2" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"]
+"lxc-ms_promote_0 lxc2" -> "lxc-ms-master_promoted_0" [ style = bold]
+"lxc-ms_promote_0 lxc2" -> "lxc-ms_monitor_10000 lxc2" [ style = bold]
+"lxc-ms_promote_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"lxc-ms_start_0 lxc2" -> "lxc-ms-master_running_0" [ style = bold]
+"lxc-ms_start_0 lxc2" -> "lxc-ms_monitor_10000 lxc2" [ style = bold]
+"lxc-ms_start_0 lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc-ms_start_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"lxc-ms_stop_0 lxc2" -> "lxc-ms-master_stopped_0" [ style = bold]
+"lxc-ms_stop_0 lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"lxc-ms_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"]
+"lxc2_monitor_30000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"lxc2_start_0 rhel7-3" -> "lxc-ms_monitor_10000 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "lxc2_monitor_30000 rhel7-3" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "rsc1_monitor_11000 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "rsc1_start_0 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "rsc2_monitor_11000 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "rsc2_start_0 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"lxc2_stop_0 rhel7-3" -> "container2_stop_0 rhel7-3" [ style = bold]
+"lxc2_stop_0 rhel7-3" -> "lxc2_start_0 rhel7-3" [ style = bold]
+"lxc2_stop_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"remote-rhel7-2_monitor_60000 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"remote-rhel7-2_start_0 rhel7-1" -> "remote-rhel7-2_monitor_60000 rhel7-1" [ style = bold]
+"remote-rhel7-2_start_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"remote-rhel7-2_stop_0 rhel7-1" -> "remote-rhel7-2_start_0 rhel7-1" [ style = bold]
+"remote-rhel7-2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_demoted_0" -> "rsc1-clone_promote_0" [ style = bold]
+"rsc1-clone_demoted_0" -> "rsc1-clone_start_0" [ style = bold]
+"rsc1-clone_demoted_0" -> "rsc1-clone_stop_0" [ style = bold]
+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promote_0" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"rsc1-clone_promote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_running_0" -> "rsc1-clone_promote_0" [ style = bold]
+"rsc1-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_start_0" -> "rsc1-clone_running_0" [ style = bold]
+"rsc1-clone_start_0" -> "rsc1_start_0 lxc2" [ style = bold]
+"rsc1-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_stop_0" -> "rsc1-clone_stopped_0" [ style = bold]
+"rsc1-clone_stop_0" -> "rsc1_stop_0 lxc2" [ style = bold]
+"rsc1-clone_stop_0" -> "rsc1_stop_0 remote-rhel7-2" [ style = bold]
+"rsc1-clone_stop_0" -> "rsc1_stop_0 rhel7-4" [ style = bold]
+"rsc1-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_stopped_0" -> "rsc1-clone_promote_0" [ style = bold]
+"rsc1-clone_stopped_0" -> "rsc1-clone_start_0" [ style = bold]
+"rsc1-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1_demote_0 rhel7-4" -> "rsc1_stop_0 rhel7-4" [ style = bold]
+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"rsc1_monitor_10000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc1_monitor_11000 lxc2" [ style=bold color="green" fontcolor="black"]
+"rsc1_promote_0 rhel7-3" -> "rsc1-clone_promoted_0" [ style = bold]
+"rsc1_promote_0 rhel7-3" -> "rsc1_monitor_10000 rhel7-3" [ style = bold]
+"rsc1_promote_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc1_start_0 lxc2" -> "rsc1-clone_running_0" [ style = bold]
+"rsc1_start_0 lxc2" -> "rsc1_monitor_11000 lxc2" [ style = bold]
+"rsc1_start_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"rsc1_stop_0 lxc2" -> "rsc1-clone_stopped_0" [ style = bold]
+"rsc1_stop_0 lxc2" -> "rsc1_start_0 lxc2" [ style = bold]
+"rsc1_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"]
+"rsc1_stop_0 remote-rhel7-2" -> "remote-rhel7-2_stop_0 rhel7-1" [ style = bold]
+"rsc1_stop_0 remote-rhel7-2" -> "rsc1-clone_stopped_0" [ style = bold]
+"rsc1_stop_0 remote-rhel7-2" [ style=bold color="green" fontcolor="orange"]
+"rsc1_stop_0 rhel7-4" -> "rsc1-clone_stopped_0" [ style = bold]
+"rsc1_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_demote_0" -> "rsc2-master_demoted_0" [ style = bold]
+"rsc2-master_demote_0" -> "rsc2_demote_0 remote-rhel7-2" [ style = bold]
+"rsc2-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_demoted_0" -> "rsc2-master_promote_0" [ style = bold]
+"rsc2-master_demoted_0" -> "rsc2-master_start_0" [ style = bold]
+"rsc2-master_demoted_0" -> "rsc2-master_stop_0" [ style = bold]
+"rsc2-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_promote_0" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"rsc2-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_running_0" -> "rsc2-master_promote_0" [ style = bold]
+"rsc2-master_running_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_start_0" -> "rsc2-master_running_0" [ style = bold]
+"rsc2-master_start_0" -> "rsc2_start_0 lxc2" [ style = bold]
+"rsc2-master_start_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_stop_0" -> "rsc2-master_stopped_0" [ style = bold]
+"rsc2-master_stop_0" -> "rsc2_stop_0 lxc2" [ style = bold]
+"rsc2-master_stop_0" -> "rsc2_stop_0 remote-rhel7-2" [ style = bold]
+"rsc2-master_stop_0" -> "rsc2_stop_0 rhel7-4" [ style = bold]
+"rsc2-master_stop_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_stopped_0" -> "rsc2-master_promote_0" [ style = bold]
+"rsc2-master_stopped_0" -> "rsc2-master_start_0" [ style = bold]
+"rsc2-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2_demote_0 remote-rhel7-2" -> "rsc2-master_demoted_0" [ style = bold]
+"rsc2_demote_0 remote-rhel7-2" -> "rsc2_stop_0 remote-rhel7-2" [ style = bold]
+"rsc2_demote_0 remote-rhel7-2" [ style=bold color="green" fontcolor="orange"]
+"rsc2_monitor_10000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc2_monitor_11000 lxc2" [ style=bold color="green" fontcolor="black"]
+"rsc2_promote_0 rhel7-3" -> "rsc2-master_promoted_0" [ style = bold]
+"rsc2_promote_0 rhel7-3" -> "rsc2_monitor_10000 rhel7-3" [ style = bold]
+"rsc2_promote_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc2_start_0 lxc2" -> "rsc2-master_running_0" [ style = bold]
+"rsc2_start_0 lxc2" -> "rsc2_monitor_11000 lxc2" [ style = bold]
+"rsc2_start_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"rsc2_stop_0 lxc2" -> "rsc2-master_stopped_0" [ style = bold]
+"rsc2_stop_0 lxc2" -> "rsc2_start_0 lxc2" [ style = bold]
+"rsc2_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"]
+"rsc2_stop_0 remote-rhel7-2" -> "remote-rhel7-2_stop_0 rhel7-1" [ style = bold]
+"rsc2_stop_0 remote-rhel7-2" -> "rsc2-master_stopped_0" [ style = bold]
+"rsc2_stop_0 remote-rhel7-2" [ style=bold color="green" fontcolor="orange"]
+"rsc2_stop_0 rhel7-4" -> "rsc2-master_stopped_0" [ style = bold]
+"rsc2_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-0_monitor_30000 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-0_start_0 rhel7-5" -> "bundled_monitor_10000 stateful-bundle-0" [ style = bold]
+"stateful-bundle-0_start_0 rhel7-5" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-0_start_0 rhel7-5" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-0_start_0 rhel7-5" -> "stateful-bundle-0_monitor_30000 rhel7-5" [ style = bold]
+"stateful-bundle-0_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-0_stop_0 rhel7-5" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold]
+"stateful-bundle-0_stop_0 rhel7-5" -> "stateful-bundle-docker-0_stop_0 rhel7-5" [ style = bold]
+"stateful-bundle-0_stop_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-2_monitor_30000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-2_start_0 rhel7-3" -> "bundled_monitor_11000 stateful-bundle-2" [ style = bold]
+"stateful-bundle-2_start_0 rhel7-3" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stateful-bundle-2_start_0 rhel7-3" -> "stateful-bundle-2_monitor_30000 rhel7-3" [ style = bold]
+"stateful-bundle-2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-2_stop_0 rhel7-4" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold]
+"stateful-bundle-2_stop_0 rhel7-4" -> "stateful-bundle-docker-2_stop_0 rhel7-4" [ style = bold]
+"stateful-bundle-2_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-docker-0_monitor_60000 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-docker-0_start_0 rhel7-5" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-docker-0_start_0 rhel7-5" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-docker-0_start_0 rhel7-5" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold]
+"stateful-bundle-docker-0_start_0 rhel7-5" -> "stateful-bundle-docker-0_monitor_60000 rhel7-5" [ style = bold]
+"stateful-bundle-docker-0_start_0 rhel7-5" -> "stateful-bundle_running_0" [ style = bold]
+"stateful-bundle-docker-0_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-docker-0_stop_0 rhel7-5" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stateful-bundle-docker-0_stop_0 rhel7-5" -> "stateful-bundle_stopped_0" [ style = bold]
+"stateful-bundle-docker-0_stop_0 rhel7-5" -> "stonith 'reboot' stateful-bundle-0" [ style = bold]
+"stateful-bundle-docker-0_stop_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-docker-2_monitor_60000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-docker-2_start_0 rhel7-3" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stateful-bundle-docker-2_start_0 rhel7-3" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold]
+"stateful-bundle-docker-2_start_0 rhel7-3" -> "stateful-bundle-docker-2_monitor_60000 rhel7-3" [ style = bold]
+"stateful-bundle-docker-2_start_0 rhel7-3" -> "stateful-bundle_running_0" [ style = bold]
+"stateful-bundle-docker-2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-docker-2_stop_0 rhel7-4" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stateful-bundle-docker-2_stop_0 rhel7-4" -> "stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" [ style = bold]
+"stateful-bundle-docker-2_stop_0 rhel7-4" -> "stateful-bundle_stopped_0" [ style = bold]
+"stateful-bundle-docker-2_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-ip-192.168.122.133_monitor_60000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" -> "stateful-bundle-ip-192.168.122.133_monitor_60000 rhel7-3" [ style = bold]
+"stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_demote_0" -> "bundled_demote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_demote_0" -> "stateful-bundle-master_demoted_0" [ style = bold]
+"stateful-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_start_0" [ style = bold]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_stop_0" [ style = bold]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle_demoted_0" [ style = bold]
+"stateful-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_promote_0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_promoted_0" -> "stateful-bundle_promoted_0" [ style = bold]
+"stateful-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_running_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle-master_running_0" -> "stateful-bundle_running_0" [ style = bold]
+"stateful-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_start_0" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_start_0" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stateful-bundle-master_start_0" -> "stateful-bundle-master_running_0" [ style = bold]
+"stateful-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_stop_0" -> "bundled_stop_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_stop_0" -> "bundled_stop_0 stateful-bundle-2" [ style = bold]
+"stateful-bundle-master_stop_0" -> "stateful-bundle-master_stopped_0" [ style = bold]
+"stateful-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_stopped_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle-master_stopped_0" -> "stateful-bundle-master_start_0" [ style = bold]
+"stateful-bundle-master_stopped_0" -> "stateful-bundle_stopped_0" [ style = bold]
+"stateful-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_demote_0" -> "stateful-bundle-master_demote_0" [ style = bold]
+"stateful-bundle_demote_0" -> "stateful-bundle_demoted_0" [ style = bold]
+"stateful-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_demoted_0" -> "stateful-bundle_promote_0" [ style = bold]
+"stateful-bundle_demoted_0" -> "stateful-bundle_start_0" [ style = bold]
+"stateful-bundle_demoted_0" -> "stateful-bundle_stop_0" [ style = bold]
+"stateful-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_promote_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_running_0" -> "stateful-bundle_promote_0" [ style = bold]
+"stateful-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_start_0" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stateful-bundle_start_0" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stateful-bundle_start_0" -> "stateful-bundle-master_start_0" [ style = bold]
+"stateful-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_stop_0" -> "bundled_stop_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle_stop_0" -> "bundled_stop_0 stateful-bundle-2" [ style = bold]
+"stateful-bundle_stop_0" -> "stateful-bundle-docker-0_stop_0 rhel7-5" [ style = bold]
+"stateful-bundle_stop_0" -> "stateful-bundle-docker-2_stop_0 rhel7-4" [ style = bold]
+"stateful-bundle_stop_0" -> "stateful-bundle-master_stop_0" [ style = bold]
+"stateful-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_stopped_0" -> "stateful-bundle_promote_0" [ style = bold]
+"stateful-bundle_stopped_0" -> "stateful-bundle_start_0" [ style = bold]
+"stateful-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"stonith 'reboot' lxc2" -> "Fencing_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' lxc2" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' lxc2" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' lxc2" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stonith 'reboot' lxc2" -> "container2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc-ms-master_stop_0" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc-ms_demote_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "remote-rhel7-2_start_0 rhel7-1" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc1-clone_stop_0" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc1_start_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc1_stop_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc2-master_stop_0" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc2_start_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc2_stop_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' lxc2" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' lxc2" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" [ style=bold color="green" fontcolor="orange"]
+"stonith 'reboot' remote-rhel7-2" -> "Fencing_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "container2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "lxc2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "remote-rhel7-2_start_0 rhel7-1" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc1-clone_stop_0" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc1_start_0 lxc2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc1_stop_0 remote-rhel7-2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc2-master_stop_0" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc2_demote_0 remote-rhel7-2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc2_start_0 lxc2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc2_stop_0 remote-rhel7-2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stonith 'reboot' rhel7-4" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" [ style=bold color="green" fontcolor="black"]
+"stonith 'reboot' rhel7-4" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "container2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc1-clone_stop_0" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc1_start_0 lxc2" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc1_stop_0 rhel7-4" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc2-master_stop_0" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc2_start_0 lxc2" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc2_stop_0 rhel7-4" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stateful-bundle-docker-2_stop_0 rhel7-4" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stonith 'reboot' stateful-bundle-2" [ style = bold]
+"stonith 'reboot' rhel7-4" [ style=bold color="green" fontcolor="black"]
+"stonith 'reboot' stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "container2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "rsc1_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "rsc2_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-master_stop_0" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" [ style=bold color="green" fontcolor="orange"]
+"stonith 'reboot' stateful-bundle-2" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "container2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "rsc1_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "rsc2_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-master_stop_0" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/cts/scheduler/on_fail_demote4.exp b/cts/scheduler/on_fail_demote4.exp
new file mode 100644
index 0000000..0789a12
--- /dev/null
+++ b/cts/scheduler/on_fail_demote4.exp
@@ -0,0 +1,1818 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="1" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="32" operation="monitor" operation_key="Fencing_monitor_120000" on_node="rhel7-5" on_node_uuid="5">
+ <primitive id="Fencing" class="stonith" type="fence_xvm"/>
+ <attributes CRM_meta_interval="120000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="120000" key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 rhel7-3 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5" pcmk_host_map="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="31" operation="start" operation_key="Fencing_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="31" operation="start" operation_key="Fencing_start_0" on_node="rhel7-5" on_node_uuid="5">
+ <primitive id="Fencing" class="stonith" type="fence_xvm"/>
+ <attributes CRM_meta_name="start" CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="60000" key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 rhel7-3 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5" pcmk_host_map="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="30" operation="stop" operation_key="Fencing_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <pseudo_event id="30" operation="stop" operation_key="Fencing_stop_0">
+ <attributes CRM_meta_name="stop" CRM_meta_timeout="60000" key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 rhel7-3 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5" pcmk_host_map="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <pseudo_event id="34" operation="stop" operation_key="rsc1_stop_0" internal_operation_key="rsc1:0_stop_0">
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="33" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="51" operation="stop" operation_key="rsc1-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <pseudo_event id="33" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0">
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="55" operation="demote" operation_key="rsc1-clone_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <rsc_op id="38" operation="monitor" operation_key="rsc1_monitor_10000" internal_operation_key="rsc1:1_monitor_10000" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="rsc1" long-id="rsc1:1" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_fail="demote" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="8" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Master" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="37" operation="promote" operation_key="rsc1_promote_0" internal_operation_key="rsc1:1_promote_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <rsc_op id="37" operation="promote" operation_key="rsc1_promote_0" internal_operation_key="rsc1:1_promote_0" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="rsc1" long-id="rsc1:1" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="2" operation="cancel" operation_key="rsc1_monitor_11000" internal_operation_key="rsc1:1_monitor_11000" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="53" operation="promote" operation_key="rsc1-clone_promote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <rsc_op id="2" operation="cancel" operation_key="rsc1_monitor_11000" internal_operation_key="rsc1:1_monitor_11000" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="rsc1" long-id="rsc1:1" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="11000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_operation="monitor" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Slave" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="8">
+ <action_set>
+ <pseudo_event id="43" operation="stop" operation_key="rsc1_stop_0" internal_operation_key="rsc1:4_stop_0">
+ <attributes CRM_meta_clone="4" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="51" operation="stop" operation_key="rsc1-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="9">
+ <action_set>
+ <rsc_op id="46" operation="monitor" operation_key="rsc1_monitor_11000" internal_operation_key="rsc1:5_monitor_11000" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3">
+ <primitive id="rsc1" long-id="rsc1:5" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="5" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="11000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="lxc2" CRM_meta_on_node_uuid="lxc2" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Slave" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="45" operation="start" operation_key="rsc1_start_0" internal_operation_key="rsc1:5_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="147" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="10">
+ <action_set>
+ <rsc_op id="45" operation="start" operation_key="rsc1_start_0" internal_operation_key="rsc1:5_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3">
+ <primitive id="rsc1" long-id="rsc1:5" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="5" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="lxc2" CRM_meta_on_node_uuid="lxc2" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="44" operation="stop" operation_key="rsc1_stop_0" internal_operation_key="rsc1:5_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="49" operation="start" operation_key="rsc1-clone_start_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="84" operation="start" operation_key="container2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="147" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="11">
+ <action_set>
+ <pseudo_event id="44" operation="stop" operation_key="rsc1_stop_0" internal_operation_key="rsc1:5_stop_0">
+ <attributes CRM_meta_clone="5" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="51" operation="stop" operation_key="rsc1-clone_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="12" priority="1000000">
+ <action_set>
+ <pseudo_event id="56" operation="demoted" operation_key="rsc1-clone_demoted_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="33" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="55" operation="demote" operation_key="rsc1-clone_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="13">
+ <action_set>
+ <pseudo_event id="55" operation="demote" operation_key="rsc1-clone_demote_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="14" priority="1000000">
+ <action_set>
+ <pseudo_event id="54" operation="promoted" operation_key="rsc1-clone_promoted_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="37" operation="promote" operation_key="rsc1_promote_0" internal_operation_key="rsc1:1_promote_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="15">
+ <action_set>
+ <pseudo_event id="53" operation="promote" operation_key="rsc1-clone_promote_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="50" operation="running" operation_key="rsc1-clone_running_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="52" operation="stopped" operation_key="rsc1-clone_stopped_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="56" operation="demoted" operation_key="rsc1-clone_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="16" priority="1000000">
+ <action_set>
+ <pseudo_event id="52" operation="stopped" operation_key="rsc1-clone_stopped_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="34" operation="stop" operation_key="rsc1_stop_0" internal_operation_key="rsc1:0_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="43" operation="stop" operation_key="rsc1_stop_0" internal_operation_key="rsc1:4_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="44" operation="stop" operation_key="rsc1_stop_0" internal_operation_key="rsc1:5_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="51" operation="stop" operation_key="rsc1-clone_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="17">
+ <action_set>
+ <pseudo_event id="51" operation="stop" operation_key="rsc1-clone_stop_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="56" operation="demoted" operation_key="rsc1-clone_demoted_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="18" priority="1000000">
+ <action_set>
+ <pseudo_event id="50" operation="running" operation_key="rsc1-clone_running_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="45" operation="start" operation_key="rsc1_start_0" internal_operation_key="rsc1:5_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="49" operation="start" operation_key="rsc1-clone_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="19">
+ <action_set>
+ <pseudo_event id="49" operation="start" operation_key="rsc1-clone_start_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="52" operation="stopped" operation_key="rsc1-clone_stopped_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="56" operation="demoted" operation_key="rsc1-clone_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="20">
+ <action_set>
+ <pseudo_event id="57" operation="stop" operation_key="rsc2_stop_0" internal_operation_key="rsc2:0_stop_0">
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="75" operation="stop" operation_key="rsc2-master_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="21">
+ <action_set>
+ <rsc_op id="61" operation="monitor" operation_key="rsc2_monitor_10000" internal_operation_key="rsc2:1_monitor_10000" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="rsc2" long-id="rsc2:1" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_fail="demote" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="8" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Master" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="60" operation="promote" operation_key="rsc2_promote_0" internal_operation_key="rsc2:1_promote_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="22">
+ <action_set>
+ <rsc_op id="60" operation="promote" operation_key="rsc2_promote_0" internal_operation_key="rsc2:1_promote_0" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="rsc2" long-id="rsc2:1" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="3" operation="cancel" operation_key="rsc2_monitor_11000" internal_operation_key="rsc2:1_monitor_11000" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="77" operation="promote" operation_key="rsc2-master_promote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="23">
+ <action_set>
+ <rsc_op id="3" operation="cancel" operation_key="rsc2_monitor_11000" internal_operation_key="rsc2:1_monitor_11000" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="rsc2" long-id="rsc2:1" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="11000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_operation="monitor" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Slave" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="24">
+ <action_set>
+ <pseudo_event id="67" operation="stop" operation_key="rsc2_stop_0" internal_operation_key="rsc2:4_stop_0">
+ <attributes CRM_meta_clone="4" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="66" operation="demote" operation_key="rsc2_demote_0" internal_operation_key="rsc2:4_demote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="75" operation="stop" operation_key="rsc2-master_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="25">
+ <action_set>
+ <pseudo_event id="66" operation="demote" operation_key="rsc2_demote_0" internal_operation_key="rsc2:4_demote_0">
+ <attributes CRM_meta_clone="4" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="79" operation="demote" operation_key="rsc2-master_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="26">
+ <action_set>
+ <rsc_op id="70" operation="monitor" operation_key="rsc2_monitor_11000" internal_operation_key="rsc2:5_monitor_11000" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3">
+ <primitive id="rsc2" long-id="rsc2:5" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="5" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="11000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="lxc2" CRM_meta_on_node_uuid="lxc2" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Slave" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="69" operation="start" operation_key="rsc2_start_0" internal_operation_key="rsc2:5_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="147" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="27">
+ <action_set>
+ <rsc_op id="69" operation="start" operation_key="rsc2_start_0" internal_operation_key="rsc2:5_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3">
+ <primitive id="rsc2" long-id="rsc2:5" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="5" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="lxc2" CRM_meta_on_node_uuid="lxc2" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="68" operation="stop" operation_key="rsc2_stop_0" internal_operation_key="rsc2:5_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="73" operation="start" operation_key="rsc2-master_start_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="84" operation="start" operation_key="container2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="147" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="28">
+ <action_set>
+ <pseudo_event id="68" operation="stop" operation_key="rsc2_stop_0" internal_operation_key="rsc2:5_stop_0">
+ <attributes CRM_meta_clone="5" CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="75" operation="stop" operation_key="rsc2-master_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="29" priority="1000000">
+ <action_set>
+ <pseudo_event id="80" operation="demoted" operation_key="rsc2-master_demoted_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="66" operation="demote" operation_key="rsc2_demote_0" internal_operation_key="rsc2:4_demote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="79" operation="demote" operation_key="rsc2-master_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="30">
+ <action_set>
+ <pseudo_event id="79" operation="demote" operation_key="rsc2-master_demote_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="31" priority="1000000">
+ <action_set>
+ <pseudo_event id="78" operation="promoted" operation_key="rsc2-master_promoted_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="60" operation="promote" operation_key="rsc2_promote_0" internal_operation_key="rsc2:1_promote_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="32">
+ <action_set>
+ <pseudo_event id="77" operation="promote" operation_key="rsc2-master_promote_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="74" operation="running" operation_key="rsc2-master_running_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="76" operation="stopped" operation_key="rsc2-master_stopped_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="80" operation="demoted" operation_key="rsc2-master_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="33" priority="1000000">
+ <action_set>
+ <pseudo_event id="76" operation="stopped" operation_key="rsc2-master_stopped_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="57" operation="stop" operation_key="rsc2_stop_0" internal_operation_key="rsc2:0_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="67" operation="stop" operation_key="rsc2_stop_0" internal_operation_key="rsc2:4_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="68" operation="stop" operation_key="rsc2_stop_0" internal_operation_key="rsc2:5_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="75" operation="stop" operation_key="rsc2-master_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="34">
+ <action_set>
+ <pseudo_event id="75" operation="stop" operation_key="rsc2-master_stop_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="80" operation="demoted" operation_key="rsc2-master_demoted_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="35" priority="1000000">
+ <action_set>
+ <pseudo_event id="74" operation="running" operation_key="rsc2-master_running_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="69" operation="start" operation_key="rsc2_start_0" internal_operation_key="rsc2:5_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="73" operation="start" operation_key="rsc2-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="36">
+ <action_set>
+ <pseudo_event id="73" operation="start" operation_key="rsc2-master_start_0">
+ <attributes CRM_meta_clone_max="7" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="76" operation="stopped" operation_key="rsc2-master_stopped_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="80" operation="demoted" operation_key="rsc2-master_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="37">
+ <action_set>
+ <rsc_op id="81" operation="start" operation_key="remote-rhel7-2_start_0" on_node="rhel7-1" on_node_uuid="1">
+ <primitive id="remote-rhel7-2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_name="start" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="120000" server="rhel7-2"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="22" operation="stop" operation_key="remote-rhel7-2_stop_0" on_node="rhel7-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="38">
+ <action_set>
+ <rsc_op id="22" operation="stop" operation_key="remote-rhel7-2_stop_0" on_node="rhel7-1" on_node_uuid="1">
+ <primitive id="remote-rhel7-2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="90000" server="rhel7-2"/>
+ <downed>
+ <node id="remote-rhel7-2"/>
+ </downed>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="43" operation="stop" operation_key="rsc1_stop_0" internal_operation_key="rsc1:4_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="67" operation="stop" operation_key="rsc2_stop_0" internal_operation_key="rsc2:4_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="39">
+ <action_set>
+ <rsc_op id="20" operation="monitor" operation_key="remote-rhel7-2_monitor_60000" on_node="rhel7-1" on_node_uuid="1">
+ <primitive id="remote-rhel7-2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="90000" server="rhel7-2"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="81" operation="start" operation_key="remote-rhel7-2_start_0" on_node="rhel7-1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="40">
+ <action_set>
+ <rsc_op id="84" operation="start" operation_key="container2_start_0" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="container2" class="ocf" provider="heartbeat" type="VirtualDomain"/>
+ <attributes CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_remote_node="lxc2" CRM_meta_timeout="90000" config="/var/lib/pacemaker/cts/lxc/lxc2.xml" force_stop="true" hypervisor="lxc:///"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="6" operation="stop" operation_key="container2_stop_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="41">
+ <action_set>
+ <rsc_op id="6" operation="stop" operation_key="container2_stop_0" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="container2" class="ocf" provider="heartbeat" type="VirtualDomain"/>
+ <attributes CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_remote_node="lxc2" CRM_meta_timeout="90000" config="/var/lib/pacemaker/cts/lxc/lxc2.xml" force_stop="true" hypervisor="lxc:///"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="146" operation="stop" operation_key="lxc2_stop_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="42">
+ <action_set>
+ <rsc_op id="5" operation="monitor" operation_key="container2_monitor_20000" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="container2" class="ocf" provider="heartbeat" type="VirtualDomain"/>
+ <attributes CRM_meta_interval="20000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_remote_node="lxc2" CRM_meta_timeout="90000" config="/var/lib/pacemaker/cts/lxc/lxc2.xml" force_stop="true" hypervisor="lxc:///"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="84" operation="start" operation_key="container2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="43">
+ <action_set>
+ <rsc_op id="88" operation="monitor" operation_key="lxc-ms_monitor_10000" internal_operation_key="lxc-ms:0_monitor_10000" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3">
+ <primitive id="lxc-ms" long-id="lxc-ms:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_fail="demote" CRM_meta_on_node="lxc2" CRM_meta_on_node_uuid="lxc2" CRM_meta_op_target_rc="8" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Master" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="86" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:0_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="87" operation="promote" operation_key="lxc-ms_promote_0" internal_operation_key="lxc-ms:0_promote_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="147" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="44">
+ <action_set>
+ <rsc_op id="87" operation="promote" operation_key="lxc-ms_promote_0" internal_operation_key="lxc-ms:0_promote_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3">
+ <primitive id="lxc-ms" long-id="lxc-ms:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="lxc2" CRM_meta_on_node_uuid="lxc2" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="23" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="84" operation="start" operation_key="container2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="86" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:0_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="95" operation="promote" operation_key="lxc-ms-master_promote_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="147" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="45">
+ <action_set>
+ <rsc_op id="86" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:0_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3">
+ <primitive id="lxc-ms" long-id="lxc-ms:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="lxc2" CRM_meta_on_node_uuid="lxc2" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="84" operation="start" operation_key="container2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="85" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:0_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="91" operation="start" operation_key="lxc-ms-master_start_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="147" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="46">
+ <action_set>
+ <pseudo_event id="85" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:0_stop_0">
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="23" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="93" operation="stop" operation_key="lxc-ms-master_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="47">
+ <action_set>
+ <pseudo_event id="23" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0">
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="97" operation="demote" operation_key="lxc-ms-master_demote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="48" priority="1000000">
+ <action_set>
+ <pseudo_event id="98" operation="demoted" operation_key="lxc-ms-master_demoted_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="23" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="97" operation="demote" operation_key="lxc-ms-master_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="49">
+ <action_set>
+ <pseudo_event id="97" operation="demote" operation_key="lxc-ms-master_demote_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="50" priority="1000000">
+ <action_set>
+ <pseudo_event id="96" operation="promoted" operation_key="lxc-ms-master_promoted_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="87" operation="promote" operation_key="lxc-ms_promote_0" internal_operation_key="lxc-ms:0_promote_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="51">
+ <action_set>
+ <pseudo_event id="95" operation="promote" operation_key="lxc-ms-master_promote_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="92" operation="running" operation_key="lxc-ms-master_running_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="94" operation="stopped" operation_key="lxc-ms-master_stopped_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="98" operation="demoted" operation_key="lxc-ms-master_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="52" priority="1000000">
+ <action_set>
+ <pseudo_event id="94" operation="stopped" operation_key="lxc-ms-master_stopped_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="85" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:0_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="93" operation="stop" operation_key="lxc-ms-master_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="53">
+ <action_set>
+ <pseudo_event id="93" operation="stop" operation_key="lxc-ms-master_stop_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="98" operation="demoted" operation_key="lxc-ms-master_demoted_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="54" priority="1000000">
+ <action_set>
+ <pseudo_event id="92" operation="running" operation_key="lxc-ms-master_running_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="86" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:0_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="91" operation="start" operation_key="lxc-ms-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="55">
+ <action_set>
+ <pseudo_event id="91" operation="start" operation_key="lxc-ms-master_start_0">
+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="94" operation="stopped" operation_key="lxc-ms-master_stopped_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="98" operation="demoted" operation_key="lxc-ms-master_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="56">
+ <action_set>
+ <rsc_op id="126" operation="monitor" operation_key="bundled_monitor_10000" internal_operation_key="bundled:0_monitor_10000" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5">
+ <primitive id="bundled" long-id="bundled:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_fail="demote" CRM_meta_on_node="stateful-bundle-0" CRM_meta_on_node_uuid="stateful-bundle-0" CRM_meta_op_target_rc="8" CRM_meta_physical_host="rhel7-5" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Master" CRM_meta_timeout="20000" pcmk_external_ip="192.168.122.131"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="103" operation="start" operation_key="stateful-bundle-0_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="124" operation="start" operation_key="bundled_start_0" internal_operation_key="bundled:0_start_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="125" operation="promote" operation_key="bundled_promote_0" internal_operation_key="bundled:0_promote_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="57">
+ <action_set>
+ <rsc_op id="125" operation="promote" operation_key="bundled_promote_0" internal_operation_key="bundled:0_promote_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5">
+ <primitive id="bundled" long-id="bundled:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="stateful-bundle-0" CRM_meta_on_node_uuid="stateful-bundle-0" CRM_meta_physical_host="rhel7-5" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" pcmk_external_ip="192.168.122.131"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="28" operation="demote" operation_key="bundled_demote_0" internal_operation_key="bundled:0_demote_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="101" operation="start" operation_key="stateful-bundle-docker-0_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="103" operation="start" operation_key="stateful-bundle-0_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="124" operation="start" operation_key="bundled_start_0" internal_operation_key="bundled:0_start_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="136" operation="promote" operation_key="stateful-bundle-master_promote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="58">
+ <action_set>
+ <rsc_op id="124" operation="start" operation_key="bundled_start_0" internal_operation_key="bundled:0_start_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5">
+ <primitive id="bundled" long-id="bundled:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="stateful-bundle-0" CRM_meta_on_node_uuid="stateful-bundle-0" CRM_meta_physical_host="rhel7-5" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" pcmk_external_ip="192.168.122.131"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="101" operation="start" operation_key="stateful-bundle-docker-0_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="103" operation="start" operation_key="stateful-bundle-0_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="123" operation="stop" operation_key="bundled_stop_0" internal_operation_key="bundled:0_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="132" operation="start" operation_key="stateful-bundle-master_start_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="59">
+ <action_set>
+ <pseudo_event id="123" operation="stop" operation_key="bundled_stop_0" internal_operation_key="bundled:0_stop_0">
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_physical_host="rhel7-5" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" pcmk_external_ip="192.168.122.131"/>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="28" operation="demote" operation_key="bundled_demote_0" internal_operation_key="bundled:0_demote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="121" operation="stop" operation_key="stateful-bundle_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="129" operation="stop" operation_key="bundled_stop_0" internal_operation_key="bundled:2_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="134" operation="stop" operation_key="stateful-bundle-master_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="60">
+ <action_set>
+ <pseudo_event id="28" operation="demote" operation_key="bundled_demote_0" internal_operation_key="bundled:0_demote_0">
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_physical_host="rhel7-5" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" pcmk_external_ip="192.168.122.131"/>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="138" operation="demote" operation_key="stateful-bundle-master_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="61">
+ <action_set>
+ <rsc_op id="131" operation="monitor" operation_key="bundled_monitor_11000" internal_operation_key="bundled:2_monitor_11000" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2" router_node="rhel7-3">
+ <primitive id="bundled" long-id="bundled:2" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="11000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="stateful-bundle-2" CRM_meta_on_node_uuid="stateful-bundle-2" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Slave" CRM_meta_timeout="20000" pcmk_external_ip="192.168.122.133"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="117" operation="start" operation_key="stateful-bundle-2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="130" operation="start" operation_key="bundled_start_0" internal_operation_key="bundled:2_start_0" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2" router_node="rhel7-3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="62">
+ <action_set>
+ <rsc_op id="130" operation="start" operation_key="bundled_start_0" internal_operation_key="bundled:2_start_0" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2" router_node="rhel7-3">
+ <primitive id="bundled" long-id="bundled:2" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="stateful-bundle-2" CRM_meta_on_node_uuid="stateful-bundle-2" CRM_meta_physical_host="rhel7-3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" pcmk_external_ip="192.168.122.133"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="114" operation="start" operation_key="stateful-bundle-docker-2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="117" operation="start" operation_key="stateful-bundle-2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="124" operation="start" operation_key="bundled_start_0" internal_operation_key="bundled:0_start_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="129" operation="stop" operation_key="bundled_stop_0" internal_operation_key="bundled:2_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="132" operation="start" operation_key="stateful-bundle-master_start_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="63">
+ <action_set>
+ <pseudo_event id="129" operation="stop" operation_key="bundled_stop_0" internal_operation_key="bundled:2_stop_0">
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_physical_host="rhel7-4" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" pcmk_external_ip="192.168.122.133"/>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="121" operation="stop" operation_key="stateful-bundle_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="134" operation="stop" operation_key="stateful-bundle-master_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="64" priority="1000000">
+ <action_set>
+ <pseudo_event id="139" operation="demoted" operation_key="stateful-bundle-master_demoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="28" operation="demote" operation_key="bundled_demote_0" internal_operation_key="bundled:0_demote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="138" operation="demote" operation_key="stateful-bundle-master_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="65">
+ <action_set>
+ <pseudo_event id="138" operation="demote" operation_key="stateful-bundle-master_demote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="142" operation="demote" operation_key="stateful-bundle_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="66" priority="1000000">
+ <action_set>
+ <pseudo_event id="137" operation="promoted" operation_key="stateful-bundle-master_promoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="125" operation="promote" operation_key="bundled_promote_0" internal_operation_key="bundled:0_promote_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="67">
+ <action_set>
+ <pseudo_event id="136" operation="promote" operation_key="stateful-bundle-master_promote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="133" operation="running" operation_key="stateful-bundle-master_running_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="135" operation="stopped" operation_key="stateful-bundle-master_stopped_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="139" operation="demoted" operation_key="stateful-bundle-master_demoted_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="140" operation="promote" operation_key="stateful-bundle_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="68" priority="1000000">
+ <action_set>
+ <pseudo_event id="135" operation="stopped" operation_key="stateful-bundle-master_stopped_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="123" operation="stop" operation_key="bundled_stop_0" internal_operation_key="bundled:0_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="129" operation="stop" operation_key="bundled_stop_0" internal_operation_key="bundled:2_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="134" operation="stop" operation_key="stateful-bundle-master_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="69">
+ <action_set>
+ <pseudo_event id="134" operation="stop" operation_key="stateful-bundle-master_stop_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="121" operation="stop" operation_key="stateful-bundle_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="139" operation="demoted" operation_key="stateful-bundle-master_demoted_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="70" priority="1000000">
+ <action_set>
+ <pseudo_event id="133" operation="running" operation_key="stateful-bundle-master_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="124" operation="start" operation_key="bundled_start_0" internal_operation_key="bundled:0_start_0" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0" router_node="rhel7-5"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="130" operation="start" operation_key="bundled_start_0" internal_operation_key="bundled:2_start_0" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2" router_node="rhel7-3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="132" operation="start" operation_key="stateful-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="71">
+ <action_set>
+ <pseudo_event id="132" operation="start" operation_key="stateful-bundle-master_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="119" operation="start" operation_key="stateful-bundle_start_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="135" operation="stopped" operation_key="stateful-bundle-master_stopped_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="139" operation="demoted" operation_key="stateful-bundle-master_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="72">
+ <action_set>
+ <rsc_op id="101" operation="start" operation_key="stateful-bundle-docker-0_start_0" on_node="rhel7-5" on_node_uuid="5">
+ <primitive id="stateful-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="90000" allow_pull="true" force_kill="false" image="pcmktest:http" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/stateful-bundle-0" reuse="false" run_cmd="/usr/sbin/pacemaker-remoted" run_opts=" --restart=no -h stateful-bundle-0 -e PCMK_stderr=1 -e PCMK_remote_port=9999 -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/stateful-bundle-0:/var/log -p 192.168.122.131:80:80 -p 192.168.122.131:9999:9999 --log-driver=journald --add-host=stateful-bundle-0:192.168.122.131 --add-host=stateful-bundle-1:192.168.122.132 --add-host=stateful-bundle-2:192.168.122.133"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="13" operation="stop" operation_key="stateful-bundle-docker-0_stop_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="119" operation="start" operation_key="stateful-bundle_start_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="73">
+ <action_set>
+ <rsc_op id="13" operation="stop" operation_key="stateful-bundle-docker-0_stop_0" on_node="rhel7-5" on_node_uuid="5">
+ <primitive id="stateful-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="90000" allow_pull="true" force_kill="false" image="pcmktest:http" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/stateful-bundle-0" reuse="false" run_cmd="/usr/sbin/pacemaker-remoted" run_opts=" --restart=no -h stateful-bundle-0 -e PCMK_stderr=1 -e PCMK_remote_port=9999 -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/stateful-bundle-0:/var/log -p 192.168.122.131:80:80 -p 192.168.122.131:9999:9999 --log-driver=journald --add-host=stateful-bundle-0:192.168.122.131 --add-host=stateful-bundle-1:192.168.122.132 --add-host=stateful-bundle-2:192.168.122.133"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="102" operation="stop" operation_key="stateful-bundle-0_stop_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="121" operation="stop" operation_key="stateful-bundle_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="74">
+ <action_set>
+ <rsc_op id="12" operation="monitor" operation_key="stateful-bundle-docker-0_monitor_60000" on_node="rhel7-5" on_node_uuid="5">
+ <primitive id="stateful-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="90000" allow_pull="true" force_kill="false" image="pcmktest:http" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/stateful-bundle-0" reuse="false" run_cmd="/usr/sbin/pacemaker-remoted" run_opts=" --restart=no -h stateful-bundle-0 -e PCMK_stderr=1 -e PCMK_remote_port=9999 -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/stateful-bundle-0:/var/log -p 192.168.122.131:80:80 -p 192.168.122.131:9999:9999 --log-driver=journald --add-host=stateful-bundle-0:192.168.122.131 --add-host=stateful-bundle-1:192.168.122.132 --add-host=stateful-bundle-2:192.168.122.133"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="101" operation="start" operation_key="stateful-bundle-docker-0_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="75">
+ <action_set>
+ <rsc_op id="103" operation="start" operation_key="stateful-bundle-0_start_0" on_node="rhel7-5" on_node_uuid="5">
+ <primitive id="stateful-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="stateful-bundle-docker-0" CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="90000" addr="192.168.122.131" port="9999"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="101" operation="start" operation_key="stateful-bundle-docker-0_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="102" operation="stop" operation_key="stateful-bundle-0_stop_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="76">
+ <action_set>
+ <rsc_op id="102" operation="stop" operation_key="stateful-bundle-0_stop_0" on_node="rhel7-5" on_node_uuid="5">
+ <primitive id="stateful-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="stateful-bundle-docker-0" CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="90000" addr="192.168.122.131" port="9999"/>
+ <downed>
+ <node id="stateful-bundle-0"/>
+ </downed>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="77">
+ <action_set>
+ <rsc_op id="11" operation="monitor" operation_key="stateful-bundle-0_monitor_30000" on_node="rhel7-5" on_node_uuid="5">
+ <primitive id="stateful-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="stateful-bundle-docker-0" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-5" CRM_meta_on_node_uuid="5" CRM_meta_timeout="30000" addr="192.168.122.131" port="9999"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="103" operation="start" operation_key="stateful-bundle-0_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="78">
+ <action_set>
+ <rsc_op id="112" operation="monitor" operation_key="stateful-bundle-ip-192.168.122.133_monitor_60000" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="stateful-bundle-ip-192.168.122.133" class="ocf" provider="heartbeat" type="IPaddr2"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="90000" cidr_netmask="24" ip="192.168.122.133" nic="eth0"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="111" operation="start" operation_key="stateful-bundle-ip-192.168.122.133_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="79">
+ <action_set>
+ <rsc_op id="111" operation="start" operation_key="stateful-bundle-ip-192.168.122.133_start_0" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="stateful-bundle-ip-192.168.122.133" class="ocf" provider="heartbeat" type="IPaddr2"/>
+ <attributes CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="90000" cidr_netmask="24" ip="192.168.122.133" nic="eth0"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="110" operation="stop" operation_key="stateful-bundle-ip-192.168.122.133_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="80">
+ <action_set>
+ <pseudo_event id="110" operation="stop" operation_key="stateful-bundle-ip-192.168.122.133_stop_0">
+ <attributes CRM_meta_timeout="90000" cidr_netmask="24" ip="192.168.122.133" nic="eth0"/>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="113" operation="stop" operation_key="stateful-bundle-docker-2_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="81">
+ <action_set>
+ <rsc_op id="115" operation="monitor" operation_key="stateful-bundle-docker-2_monitor_60000" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="stateful-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="90000" allow_pull="true" force_kill="false" image="pcmktest:http" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/stateful-bundle-2" reuse="false" run_cmd="/usr/sbin/pacemaker-remoted" run_opts=" --restart=no -h stateful-bundle-2 -e PCMK_stderr=1 -e PCMK_remote_port=9999 -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/stateful-bundle-2:/var/log -p 192.168.122.133:80:80 -p 192.168.122.133:9999:9999 --log-driver=journald --add-host=stateful-bundle-0:192.168.122.131 --add-host=stateful-bundle-1:192.168.122.132 --add-host=stateful-bundle-2:192.168.122.133"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="114" operation="start" operation_key="stateful-bundle-docker-2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="82">
+ <action_set>
+ <rsc_op id="114" operation="start" operation_key="stateful-bundle-docker-2_start_0" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="stateful-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="90000" allow_pull="true" force_kill="false" image="pcmktest:http" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/stateful-bundle-2" reuse="false" run_cmd="/usr/sbin/pacemaker-remoted" run_opts=" --restart=no -h stateful-bundle-2 -e PCMK_stderr=1 -e PCMK_remote_port=9999 -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/stateful-bundle-2:/var/log -p 192.168.122.133:80:80 -p 192.168.122.133:9999:9999 --log-driver=journald --add-host=stateful-bundle-0:192.168.122.131 --add-host=stateful-bundle-1:192.168.122.132 --add-host=stateful-bundle-2:192.168.122.133"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="111" operation="start" operation_key="stateful-bundle-ip-192.168.122.133_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="113" operation="stop" operation_key="stateful-bundle-docker-2_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="119" operation="start" operation_key="stateful-bundle_start_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="83">
+ <action_set>
+ <pseudo_event id="113" operation="stop" operation_key="stateful-bundle-docker-2_stop_0">
+ <attributes CRM_meta_timeout="90000" allow_pull="true" force_kill="false" image="pcmktest:http" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/stateful-bundle-2" reuse="false" run_cmd="/usr/sbin/pacemaker-remoted" run_opts=" --restart=no -h stateful-bundle-2 -e PCMK_stderr=1 -e PCMK_remote_port=9999 -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/stateful-bundle-2:/var/log -p 192.168.122.133:80:80 -p 192.168.122.133:9999:9999 --log-driver=journald --add-host=stateful-bundle-0:192.168.122.131 --add-host=stateful-bundle-1:192.168.122.132 --add-host=stateful-bundle-2:192.168.122.133"/>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="116" operation="stop" operation_key="stateful-bundle-2_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="121" operation="stop" operation_key="stateful-bundle_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="84">
+ <action_set>
+ <rsc_op id="118" operation="monitor" operation_key="stateful-bundle-2_monitor_30000" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="stateful-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="stateful-bundle-docker-2" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="30000" addr="192.168.122.133" port="9999"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="117" operation="start" operation_key="stateful-bundle-2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="85">
+ <action_set>
+ <rsc_op id="117" operation="start" operation_key="stateful-bundle-2_start_0" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="stateful-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="stateful-bundle-docker-2" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="90000" addr="192.168.122.133" port="9999"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="114" operation="start" operation_key="stateful-bundle-docker-2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="116" operation="stop" operation_key="stateful-bundle-2_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="86">
+ <action_set>
+ <pseudo_event id="116" operation="stop" operation_key="stateful-bundle-2_stop_0">
+ <attributes CRM_meta_container="stateful-bundle-docker-2" CRM_meta_timeout="90000" addr="192.168.122.133" port="9999"/>
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="87">
+ <action_set>
+ <rsc_op id="147" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="lxc2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="container2" CRM_meta_name="start" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="60000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="84" operation="start" operation_key="container2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="146" operation="stop" operation_key="lxc2_stop_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="88">
+ <action_set>
+ <rsc_op id="146" operation="stop" operation_key="lxc2_stop_0" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="lxc2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="container2" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="90000" />
+ <downed>
+ <node id="lxc2"/>
+ </downed>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="89">
+ <action_set>
+ <rsc_op id="7" operation="monitor" operation_key="lxc2_monitor_30000" on_node="rhel7-3" on_node_uuid="3">
+ <primitive id="lxc2" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="container2" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="30000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="147" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="90">
+ <action_set>
+ <pseudo_event id="150" operation="stonith" operation_key="stonith-stateful-bundle-2-reboot" on_node="stateful-bundle-2" on_node_uuid="stateful-bundle-2">
+ <attributes CRM_meta_master_bundled="5" CRM_meta_on_node="stateful-bundle-2" CRM_meta_on_node_uuid="stateful-bundle-2" CRM_meta_stonith_action="reboot" />
+ <downed>
+ <node id="stateful-bundle-2"/>
+ </downed>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="91">
+ <action_set>
+ <pseudo_event id="149" operation="stonith" operation_key="stonith-stateful-bundle-0-reboot" on_node="stateful-bundle-0" on_node_uuid="stateful-bundle-0">
+ <attributes CRM_meta_master_bundled="10" CRM_meta_on_node="stateful-bundle-0" CRM_meta_on_node_uuid="stateful-bundle-0" CRM_meta_stonith_action="reboot" />
+ <downed>
+ <node id="stateful-bundle-0"/>
+ </downed>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="stop" operation_key="stateful-bundle-docker-0_stop_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="92">
+ <action_set>
+ <pseudo_event id="148" operation="stonith" operation_key="stonith-lxc2-reboot" on_node="lxc2" on_node_uuid="lxc2">
+ <attributes CRM_meta_master_lxc_ms="10" CRM_meta_master_rsc1="5" CRM_meta_master_rsc2="5" CRM_meta_on_node="lxc2" CRM_meta_on_node_uuid="lxc2" CRM_meta_stonith_action="reboot" />
+ <downed>
+ <node id="lxc2"/>
+ </downed>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="6" operation="stop" operation_key="container2_stop_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="93" priority="1000000">
+ <action_set>
+ <pseudo_event id="143" operation="demoted" operation_key="stateful-bundle_demoted_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="139" operation="demoted" operation_key="stateful-bundle-master_demoted_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="142" operation="demote" operation_key="stateful-bundle_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="94">
+ <action_set>
+ <pseudo_event id="142" operation="demote" operation_key="stateful-bundle_demote_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="95" priority="1000000">
+ <action_set>
+ <pseudo_event id="141" operation="promoted" operation_key="stateful-bundle_promoted_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="137" operation="promoted" operation_key="stateful-bundle-master_promoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="96">
+ <action_set>
+ <pseudo_event id="140" operation="promote" operation_key="stateful-bundle_promote_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="120" operation="running" operation_key="stateful-bundle_running_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="122" operation="stopped" operation_key="stateful-bundle_stopped_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="143" operation="demoted" operation_key="stateful-bundle_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="97" priority="1000000">
+ <action_set>
+ <pseudo_event id="122" operation="stopped" operation_key="stateful-bundle_stopped_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="13" operation="stop" operation_key="stateful-bundle-docker-0_stop_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="113" operation="stop" operation_key="stateful-bundle-docker-2_stop_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="135" operation="stopped" operation_key="stateful-bundle-master_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="98">
+ <action_set>
+ <pseudo_event id="121" operation="stop" operation_key="stateful-bundle_stop_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="143" operation="demoted" operation_key="stateful-bundle_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="99" priority="1000000">
+ <action_set>
+ <pseudo_event id="120" operation="running" operation_key="stateful-bundle_running_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="101" operation="start" operation_key="stateful-bundle-docker-0_start_0" on_node="rhel7-5" on_node_uuid="5"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="114" operation="start" operation_key="stateful-bundle-docker-2_start_0" on_node="rhel7-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="133" operation="running" operation_key="stateful-bundle-master_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="100">
+ <action_set>
+ <pseudo_event id="119" operation="start" operation_key="stateful-bundle_start_0">
+ <attributes CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="122" operation="stopped" operation_key="stateful-bundle_stopped_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="143" operation="demoted" operation_key="stateful-bundle_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="101">
+ <action_set>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2">
+ <attributes CRM_meta_master_rsc1="5" CRM_meta_master_rsc2="10" CRM_meta_on_node="remote-rhel7-2" CRM_meta_on_node_uuid="remote-rhel7-2" CRM_meta_stonith_action="reboot" />
+ <downed>
+ <node id="remote-rhel7-2"/>
+ </downed>
+ </crm_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="102">
+ <action_set>
+ <crm_event id="1" operation="stonith" operation_key="stonith-rhel7-4-reboot" on_node="rhel7-4" on_node_uuid="4">
+ <attributes CRM_meta_master_rsc1="10" CRM_meta_master_rsc2="10" CRM_meta_on_node="rhel7-4" CRM_meta_on_node_uuid="4" CRM_meta_standby="off" CRM_meta_stonith_action="reboot" />
+ <downed>
+ <node id="4"/>
+ <node id="stateful-bundle-2"/>
+ </downed>
+ </crm_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <crm_event id="21" operation="stonith" operation_key="stonith-remote-rhel7-2-reboot" on_node="remote-rhel7-2" on_node_uuid="remote-rhel7-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/cts/scheduler/on_fail_demote4.scores b/cts/scheduler/on_fail_demote4.scores
new file mode 100644
index 0000000..cde3fec
--- /dev/null
+++ b/cts/scheduler/on_fail_demote4.scores
@@ -0,0 +1,470 @@
+Allocation scores:
+Using the original execution date of: 2020-06-16 19:23:21Z
+bundled:0 promotion score on stateful-bundle-0: 10
+bundled:1 promotion score on stateful-bundle-1: 5
+bundled:2 promotion score on stateful-bundle-2: 5
+lxc-ms:0 promotion score on lxc2: INFINITY
+lxc-ms:1 promotion score on lxc1: INFINITY
+pcmk__bundle_allocate: bundled:0 allocation score on stateful-bundle-0: 501
+pcmk__bundle_allocate: bundled:1 allocation score on stateful-bundle-1: 501
+pcmk__bundle_allocate: bundled:2 allocation score on stateful-bundle-2: 501
+pcmk__bundle_allocate: stateful-bundle allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-0: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-2: -INFINITY
+pcmk__clone_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY
+pcmk__clone_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY
+pcmk__clone_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1-clone allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:0 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 1
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:1 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:2 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc1:3 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:3 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:4 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:4 allocation score on remote-rhel7-2: 1
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:5 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:5 allocation score on lxc2: 6
+pcmk__clone_allocate: rsc1:5 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:6 allocation score on lxc1: 6
+pcmk__clone_allocate: rsc1:6 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:6 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2-master allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2-master allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2-master allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:0 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 1
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:1 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:2 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc2:3 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:3 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:4 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:4 allocation score on remote-rhel7-2: 1
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:5 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:5 allocation score on lxc2: 6
+pcmk__clone_allocate: rsc2:5 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:6 allocation score on lxc1: 6
+pcmk__clone_allocate: rsc2:6 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:6 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-5: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc1: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc2: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on remote-rhel7-2: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-1: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-3: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-4: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-5: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-0: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-1: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-2: 0
+pcmk__native_allocate: Fencing allocation score on lxc1: -INFINITY
+pcmk__native_allocate: Fencing allocation score on lxc2: -INFINITY
+pcmk__native_allocate: Fencing allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0
+pcmk__native_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY
+pcmk__native_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY
+pcmk__native_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY
+pcmk__native_allocate: container1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: container1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: container1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: container1 allocation score on rhel7-1: 0
+pcmk__native_allocate: container1 allocation score on rhel7-3: INFINITY
+pcmk__native_allocate: container1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: container1 allocation score on rhel7-5: 0
+pcmk__native_allocate: container2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: container2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: container2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: container2 allocation score on rhel7-1: 0
+pcmk__native_allocate: container2 allocation score on rhel7-3: INFINITY
+pcmk__native_allocate: container2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: container2 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on lxc2: INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-1: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on lxc1: INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on lxc2: INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-1: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-3: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-4: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:1 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:2 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:2 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc1:3 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:3 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:3 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on lxc2: 6
+pcmk__native_allocate: rsc1:5 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on lxc1: 6
+pcmk__native_allocate: rsc1:6 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:6 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:1 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:2 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:2 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc2:3 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:3 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:3 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on lxc2: 6
+pcmk__native_allocate: rsc2:5 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on lxc1: 6
+pcmk__native_allocate: rsc2:6 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:6 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-5: 10000
+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-1: 10000
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-3: 10000
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: -INFINITY
+rsc1:0 promotion score on none: 0
+rsc1:1 promotion score on rhel7-3: 5
+rsc1:2 promotion score on rhel7-5: 5
+rsc1:3 promotion score on rhel7-1: 5
+rsc1:4 promotion score on none: 0
+rsc1:5 promotion score on lxc2: 5
+rsc1:6 promotion score on lxc1: 5
+rsc2:0 promotion score on none: 0
+rsc2:1 promotion score on rhel7-3: 5
+rsc2:2 promotion score on rhel7-5: 5
+rsc2:3 promotion score on rhel7-1: 5
+rsc2:4 promotion score on none: 0
+rsc2:5 promotion score on lxc2: 5
+rsc2:6 promotion score on lxc1: 5
diff --git a/cts/scheduler/on_fail_demote4.summary b/cts/scheduler/on_fail_demote4.summary
new file mode 100644
index 0000000..20520ff
--- /dev/null
+++ b/cts/scheduler/on_fail_demote4.summary
@@ -0,0 +1,187 @@
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Current cluster status:
+RemoteNode remote-rhel7-2: UNCLEAN (offline)
+Node rhel7-4 (4): UNCLEAN (offline)
+Online: [ rhel7-1 rhel7-3 rhel7-5 ]
+GuestOnline: [ lxc1:container1 stateful-bundle-1:stateful-bundle-docker-1 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-4 (UNCLEAN)
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ rsc1 (ocf::pacemaker:Stateful): Master rhel7-4 (UNCLEAN)
+ rsc1 (ocf::pacemaker:Stateful): Slave remote-rhel7-2 (UNCLEAN)
+ Slaves: [ lxc1 rhel7-1 rhel7-3 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ rsc2 (ocf::pacemaker:Stateful): Slave rhel7-4 (UNCLEAN)
+ rsc2 (ocf::pacemaker:Stateful): Master remote-rhel7-2 (UNCLEAN)
+ Slaves: [ lxc1 rhel7-1 rhel7-3 rhel7-5 ]
+ remote-rhel7-2 (ocf::pacemaker:remote): FAILED rhel7-1
+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ container2 (ocf::heartbeat:VirtualDomain): FAILED rhel7-3
+ Clone Set: lxc-ms-master [lxc-ms] (promotable)
+ Slaves: [ lxc1 ]
+ Stopped: [ remote-rhel7-2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+ Container bundle set: stateful-bundle [pcmktest:http]
+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): FAILED Master rhel7-5
+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1
+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): FAILED rhel7-4 (UNCLEAN)
+
+Transition Summary:
+ * Fence (reboot) stateful-bundle-2 (resource: stateful-bundle-docker-2) 'guest is unclean'
+ * Fence (reboot) stateful-bundle-0 (resource: stateful-bundle-docker-0) 'guest is unclean'
+ * Fence (reboot) lxc2 (resource: container2) 'guest is unclean'
+ * Fence (reboot) remote-rhel7-2 'remote connection is unrecoverable'
+ * Fence (reboot) rhel7-4 'peer is no longer part of the cluster'
+ * Move Fencing ( rhel7-4 -> rhel7-5 )
+ * Stop rsc1:0 ( Master rhel7-4 ) due to node availability
+ * Promote rsc1:1 ( Slave -> Master rhel7-3 )
+ * Stop rsc1:4 ( Slave remote-rhel7-2 ) due to node availability
+ * Recover rsc1:5 ( Slave lxc2 )
+ * Stop rsc2:0 ( Slave rhel7-4 ) due to node availability
+ * Promote rsc2:1 ( Slave -> Master rhel7-3 )
+ * Stop rsc2:4 ( Master remote-rhel7-2 ) due to node availability
+ * Recover rsc2:5 ( Slave lxc2 )
+ * Recover remote-rhel7-2 ( rhel7-1 )
+ * Recover container2 ( rhel7-3 )
+ * Recover lxc-ms:0 ( Master lxc2 )
+ * Recover stateful-bundle-docker-0 ( rhel7-5 )
+ * Restart stateful-bundle-0 ( rhel7-5 ) due to required stateful-bundle-docker-0 start
+ * Recover bundled:0 ( Master stateful-bundle-0 )
+ * Move stateful-bundle-ip-192.168.122.133 ( rhel7-4 -> rhel7-3 )
+ * Recover stateful-bundle-docker-2 ( rhel7-4 -> rhel7-3 )
+ * Move stateful-bundle-2 ( rhel7-4 -> rhel7-3 )
+ * Recover bundled:2 ( Slave stateful-bundle-2 )
+ * Restart lxc2 ( rhel7-3 ) due to required container2 start
+
+Executing cluster transition:
+ * Pseudo action: Fencing_stop_0
+ * Resource action: rsc1 cancel=11000 on rhel7-3
+ * Pseudo action: rsc1-clone_demote_0
+ * Resource action: rsc2 cancel=11000 on rhel7-3
+ * Pseudo action: rsc2-master_demote_0
+ * Pseudo action: lxc-ms-master_demote_0
+ * Resource action: stateful-bundle-0 stop on rhel7-5
+ * Pseudo action: stateful-bundle-2_stop_0
+ * Resource action: lxc2 stop on rhel7-3
+ * Pseudo action: stateful-bundle_demote_0
+ * Fencing remote-rhel7-2 (reboot)
+ * Fencing rhel7-4 (reboot)
+ * Pseudo action: rsc1_demote_0
+ * Pseudo action: rsc1-clone_demoted_0
+ * Pseudo action: rsc2_demote_0
+ * Pseudo action: rsc2-master_demoted_0
+ * Resource action: container2 stop on rhel7-3
+ * Pseudo action: stateful-bundle-master_demote_0
+ * Pseudo action: stonith-stateful-bundle-2-reboot on stateful-bundle-2
+ * Pseudo action: stonith-lxc2-reboot on lxc2
+ * Resource action: Fencing start on rhel7-5
+ * Pseudo action: rsc1-clone_stop_0
+ * Pseudo action: rsc2-master_stop_0
+ * Pseudo action: lxc-ms_demote_0
+ * Pseudo action: lxc-ms-master_demoted_0
+ * Pseudo action: lxc-ms-master_stop_0
+ * Pseudo action: bundled_demote_0
+ * Pseudo action: stateful-bundle-master_demoted_0
+ * Pseudo action: stateful-bundle_demoted_0
+ * Pseudo action: stateful-bundle_stop_0
+ * Resource action: Fencing monitor=120000 on rhel7-5
+ * Pseudo action: rsc1_stop_0
+ * Pseudo action: rsc1_stop_0
+ * Pseudo action: rsc1_stop_0
+ * Pseudo action: rsc1-clone_stopped_0
+ * Pseudo action: rsc1-clone_start_0
+ * Pseudo action: rsc2_stop_0
+ * Pseudo action: rsc2_stop_0
+ * Pseudo action: rsc2_stop_0
+ * Pseudo action: rsc2-master_stopped_0
+ * Pseudo action: rsc2-master_start_0
+ * Resource action: remote-rhel7-2 stop on rhel7-1
+ * Pseudo action: lxc-ms_stop_0
+ * Pseudo action: lxc-ms-master_stopped_0
+ * Pseudo action: lxc-ms-master_start_0
+ * Resource action: stateful-bundle-docker-0 stop on rhel7-5
+ * Pseudo action: stateful-bundle-docker-2_stop_0
+ * Pseudo action: stonith-stateful-bundle-0-reboot on stateful-bundle-0
+ * Resource action: remote-rhel7-2 start on rhel7-1
+ * Resource action: remote-rhel7-2 monitor=60000 on rhel7-1
+ * Resource action: container2 start on rhel7-3
+ * Resource action: container2 monitor=20000 on rhel7-3
+ * Pseudo action: stateful-bundle-master_stop_0
+ * Pseudo action: stateful-bundle-ip-192.168.122.133_stop_0
+ * Resource action: lxc2 start on rhel7-3
+ * Resource action: lxc2 monitor=30000 on rhel7-3
+ * Resource action: rsc1 start on lxc2
+ * Pseudo action: rsc1-clone_running_0
+ * Resource action: rsc2 start on lxc2
+ * Pseudo action: rsc2-master_running_0
+ * Resource action: lxc-ms start on lxc2
+ * Pseudo action: lxc-ms-master_running_0
+ * Pseudo action: bundled_stop_0
+ * Resource action: stateful-bundle-ip-192.168.122.133 start on rhel7-3
+ * Resource action: rsc1 monitor=11000 on lxc2
+ * Pseudo action: rsc1-clone_promote_0
+ * Resource action: rsc2 monitor=11000 on lxc2
+ * Pseudo action: rsc2-master_promote_0
+ * Pseudo action: lxc-ms-master_promote_0
+ * Pseudo action: bundled_stop_0
+ * Pseudo action: stateful-bundle-master_stopped_0
+ * Resource action: stateful-bundle-ip-192.168.122.133 monitor=60000 on rhel7-3
+ * Pseudo action: stateful-bundle_stopped_0
+ * Pseudo action: stateful-bundle_start_0
+ * Resource action: rsc1 promote on rhel7-3
+ * Pseudo action: rsc1-clone_promoted_0
+ * Resource action: rsc2 promote on rhel7-3
+ * Pseudo action: rsc2-master_promoted_0
+ * Resource action: lxc-ms promote on lxc2
+ * Pseudo action: lxc-ms-master_promoted_0
+ * Pseudo action: stateful-bundle-master_start_0
+ * Resource action: stateful-bundle-docker-0 start on rhel7-5
+ * Resource action: stateful-bundle-docker-0 monitor=60000 on rhel7-5
+ * Resource action: stateful-bundle-0 start on rhel7-5
+ * Resource action: stateful-bundle-0 monitor=30000 on rhel7-5
+ * Resource action: stateful-bundle-docker-2 start on rhel7-3
+ * Resource action: stateful-bundle-2 start on rhel7-3
+ * Resource action: rsc1 monitor=10000 on rhel7-3
+ * Resource action: rsc2 monitor=10000 on rhel7-3
+ * Resource action: lxc-ms monitor=10000 on lxc2
+ * Resource action: bundled start on stateful-bundle-0
+ * Resource action: bundled start on stateful-bundle-2
+ * Pseudo action: stateful-bundle-master_running_0
+ * Resource action: stateful-bundle-docker-2 monitor=60000 on rhel7-3
+ * Resource action: stateful-bundle-2 monitor=30000 on rhel7-3
+ * Pseudo action: stateful-bundle_running_0
+ * Resource action: bundled monitor=11000 on stateful-bundle-2
+ * Pseudo action: stateful-bundle_promote_0
+ * Pseudo action: stateful-bundle-master_promote_0
+ * Resource action: bundled promote on stateful-bundle-0
+ * Pseudo action: stateful-bundle-master_promoted_0
+ * Pseudo action: stateful-bundle_promoted_0
+ * Resource action: bundled monitor=10000 on stateful-bundle-0
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Revised cluster status:
+Online: [ rhel7-1 rhel7-3 rhel7-5 ]
+OFFLINE: [ rhel7-4 ]
+RemoteOnline: [ remote-rhel7-2 ]
+GuestOnline: [ lxc1:container1 lxc2:container2 stateful-bundle-0:stateful-bundle-docker-0 stateful-bundle-1:stateful-bundle-docker-1 stateful-bundle-2:stateful-bundle-docker-2 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-5
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Masters: [ rhel7-3 ]
+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-5 ]
+ Stopped: [ remote-rhel7-2 rhel7-4 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ rhel7-3 ]
+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-5 ]
+ Stopped: [ remote-rhel7-2 rhel7-4 ]
+ remote-rhel7-2 (ocf::pacemaker:remote): Started rhel7-1
+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ Clone Set: lxc-ms-master [lxc-ms] (promotable)
+ Masters: [ lxc2 ]
+ Slaves: [ lxc1 ]
+ Container bundle set: stateful-bundle [pcmktest:http]
+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): Master rhel7-5
+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1
+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): Slave rhel7-3
+
diff --git a/cts/scheduler/on_fail_demote4.xml b/cts/scheduler/on_fail_demote4.xml
new file mode 100644
index 0000000..eb4c4cc
--- /dev/null
+++ b/cts/scheduler/on_fail_demote4.xml
@@ -0,0 +1,625 @@
+<cib crm_feature_set="3.4.0" validate-with="pacemaker-3.4" epoch="62" num_updates="98" admin_epoch="1" cib-last-written="Tue Jun 16 14:21:19 2020" update-origin="rhel7-2" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2" execution-date="1592335401">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/>
+ <nvpair id="cts-start-failure-is-fatal" name="start-failure-is-fatal" value="false"/>
+ <nvpair id="cts-pe-input-series-max" name="pe-input-series-max" value="5000"/>
+ <nvpair id="cts-shutdown-escalation" name="shutdown-escalation" value="5min"/>
+ <nvpair id="cts-batch-limit" name="batch-limit" value="10"/>
+ <nvpair id="cts-dc-deadtime" name="dc-deadtime" value="5s"/>
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-578.6e1b582.git.el7_8-6e1b582"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="mycluster"/>
+ <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1591654576"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="rhel7-1">
+ <instance_attributes id="rhel7-1-1">
+ <nvpair id="rhel7-1-1-cts-fencing" name="cts-fencing" value="levels-and"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="rhel7-3"/>
+ <node id="4" uname="rhel7-4">
+ <instance_attributes id="nodes-4">
+ <nvpair id="nodes-4-standby" name="standby" value="off"/>
+ </instance_attributes>
+ </node>
+ <node id="5" uname="rhel7-5"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="Fencing" type="fence_xvm">
+ <meta_attributes id="Fencing-meta"/>
+ <instance_attributes id="Fencing-params">
+ <nvpair id="Fencing-key_file" name="key_file" value="/etc/pacemaker/fence_xvm.key"/>
+ <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/>
+ <nvpair id="Fencing-pcmk_host_map" name="pcmk_host_map" value="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
+ <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 rhel7-3 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5"/>
+ </instance_attributes>
+ <operations>
+ <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/>
+ <op id="Fencing-stop-0" interval="0" name="stop" timeout="60s"/>
+ <op id="Fencing-start-0" interval="0" name="start" timeout="60s"/>
+ </operations>
+ </primitive>
+ <clone id="rsc1-clone">
+ <meta_attributes id="rsc1-meta_attributes">
+ <nvpair id="rsc1-meta_attributes-promotable" name="promotable" value="true"/>
+ </meta_attributes>
+ <primitive class="ocf" id="rsc1" provider="pacemaker" type="Stateful">
+ <operations>
+ <op id="rsc1-demote-interval-0s" interval="0s" name="demote" timeout="10s"/>
+ <op id="rsc1-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="rsc1-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ <op id="rsc1-notify-interval-0s" interval="0s" name="notify" timeout="5s"/>
+ <op id="rsc1-promote-interval-0s" interval="0s" name="promote" timeout="10s"/>
+ <op id="rsc1-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+ <op id="rsc1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+ </operations>
+ </primitive>
+ </clone>
+ <clone id="rsc2-master">
+ <meta_attributes id="rsc2-meta_attributes">
+ <nvpair id="rsc2-meta_attributes-promotable" name="promotable" value="true"/>
+ </meta_attributes>
+ <primitive class="ocf" id="rsc2" provider="pacemaker" type="Stateful">
+ <operations>
+ <op id="rsc2-demote-interval-0s" interval="0s" name="demote" timeout="10s"/>
+ <op id="rsc2-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="rsc2-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ <op id="rsc2-notify-interval-0s" interval="0s" name="notify" timeout="5s"/>
+ <op id="rsc2-promote-interval-0s" interval="0s" name="promote" timeout="10s"/>
+ <op id="rsc2-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+ <op id="rsc2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+ </operations>
+ </primitive>
+ </clone>
+ <primitive class="ocf" id="remote-rhel7-2" provider="pacemaker" type="remote">
+ <instance_attributes id="remote-instance_attributes">
+ <nvpair id="remote-instance_attributes-server" name="server" value="rhel7-2"/>
+ </instance_attributes>
+ <operations>
+ <op id="remote-monitor-interval-60s" interval="60s" name="monitor"/>
+ <op id="remote-name-start-interval-0-timeout-120" interval="0" name="start" timeout="120"/>
+ </operations>
+ </primitive>
+ <primitive class="ocf" id="container1" provider="heartbeat" type="VirtualDomain">
+ <instance_attributes id="container1-instance_attributes">
+ <nvpair id="container1-instance_attributes-force_stop" name="force_stop" value="true"/>
+ <nvpair id="container1-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
+ <nvpair id="container1-instance_attributes-config" name="config" value="/var/lib/pacemaker/cts/lxc/lxc1.xml"/>
+ </instance_attributes>
+ <meta_attributes id="container1-meta_attributes">
+ <nvpair id="container1-meta_attributes-remote-node" name="remote-node" value="lxc1"/>
+ </meta_attributes>
+ <operations>
+ <op id="container1-monitor-20s" interval="20s" name="monitor"/>
+ </operations>
+ </primitive>
+ <primitive class="ocf" id="container2" provider="heartbeat" type="VirtualDomain">
+ <instance_attributes id="container2-instance_attributes">
+ <nvpair id="container2-instance_attributes-force_stop" name="force_stop" value="true"/>
+ <nvpair id="container2-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
+ <nvpair id="container2-instance_attributes-config" name="config" value="/var/lib/pacemaker/cts/lxc/lxc2.xml"/>
+ </instance_attributes>
+ <meta_attributes id="container2-meta_attributes">
+ <nvpair id="container2-meta_attributes-remote-node" name="remote-node" value="lxc2"/>
+ </meta_attributes>
+ <operations>
+ <op id="container2-monitor-20s" interval="20s" name="monitor"/>
+ </operations>
+ </primitive>
+ <clone id="lxc-ms-master">
+ <meta_attributes id="lxc-ms-meta_attributes">
+ <nvpair id="lxc-ms-meta_attributes-promotable" name="promotable" value="true"/>
+ <nvpair id="lxc-ms-meta_attributes-master-max" name="promoted-max" value="1"/>
+ <nvpair id="lxc-ms-meta_attributes-clone-max" name="clone-max" value="2"/>
+ </meta_attributes>
+ <primitive class="ocf" id="lxc-ms" provider="pacemaker" type="Stateful">
+ <instance_attributes id="lxc-ms-instance_attributes"/>
+ <operations>
+ <op id="lxc-ms-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="lxc-ms-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ </operations>
+ </primitive>
+ </clone>
+ <bundle id="stateful-bundle">
+ <docker image="pcmktest:http" replicas="3" promoted-max="1" options="--log-driver=journald"/>
+ <network ip-range-start="192.168.122.131" host-interface="eth0" host-netmask="24" control-port="9999">
+ <port-mapping id="bundled-port" port="80"/>
+ </network>
+ <primitive class="ocf" id="bundled" provider="pacemaker" type="Stateful">
+ <operations>
+ <op id="bundled-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="bundled-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ </operations>
+ </primitive>
+ </bundle>
+ </resources>
+ <constraints>
+ <rsc_location id="location-rsc1-clone" rsc="rsc1-clone">
+ <rule id="location-rsc1-clone-rule" role="master" score="-INFINITY">
+ <expression attribute="fail-count-rsc1#monitor_10000" id="location-rsc1-clone-rule-expr" operation="gt" value="2"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="location-rsc2-master" rsc="rsc2-master">
+ <rule id="location-rsc2-master-rule" role="master" score="100">
+ <expression id="location-rsc2-master-rule-expr" attribute="#uname" operation="eq" value="remote-rhel7-2"/>
+ </rule>
+ </rsc_location>
+ <rsc_location id="cli-prefer-container1" rsc="container1" role="Started" node="rhel7-3" score="INFINITY"/>
+ <rsc_location id="cli-prefer-container2" rsc="container2" role="Started" node="rhel7-3" score="INFINITY"/>
+ <rsc_location id="lxc-ms-location-lxc1" node="lxc1" rsc="lxc-ms-master" score="INFINITY"/>
+ <rsc_location id="lxc-ms-location-lxc2" node="lxc2" rsc="lxc-ms-master" score="INFINITY"/>
+ </constraints>
+ <op_defaults>
+ <meta_attributes id="cts-op_defaults-meta">
+ <nvpair id="cts-op_defaults-timeout" name="timeout" value="90s"/>
+ </meta_attributes>
+ </op_defaults>
+ <alerts>
+ <alert id="alert-1" path="/var/lib/pacemaker/notify.sh">
+ <recipient id="alert-1-recipient-1" value="/run/crm/alert.log"/>
+ </alert>
+ </alerts>
+ </configuration>
+ <status>
+ <node_state id="4" uname="rhel7-4" in_ccm="false" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="28:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;28:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="38" rc-code="0" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="148" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="29:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;29:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="47" rc-code="0" op-status="0" interval="120000" last-rc-change="1592342392" exec-time="65" queue-time="0" op-digest="cb34bc19df153021ce8f301baa293f35"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="6:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;6:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="18" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="94" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_10000" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="12:2:8:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:8;12:2:8:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="20" rc-code="8" op-status="0" interval="10000" last-rc-change="1592335281" exec-time="26" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="rsc1_last_failure_0" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="12:2:8:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;12:2:8:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-4" call-id="20" rc-code="7" op-status="0" interval="10000" last-rc-change="1592335401" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_demote_0" operation="demote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="37:0:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" transition-magic="0:0;37:0:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" exit-reason="" on_node="rhel7-4" call-id="49" rc-code="0" op-status="0" interval="0" last-rc-change="1592340727" last-run="1592340727" exec-time="55" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="36:1:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" transition-magic="0:0;36:1:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" exit-reason="" on_node="rhel7-4" call-id="51" rc-code="0" op-status="0" interval="11000" last-rc-change="1592340727" exec-time="19" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="remote-rhel7-2" type="remote" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="remote-rhel7-2_last_0" operation_key="remote-rhel7-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="14:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:7;14:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-4" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1592340505" last-run="1592340505" exec-time="0" queue-time="0" op-digest="f1c162fde9f781a7f852b761a6e079e5" op-force-restart=" server " op-restart-digest="f1c162fde9f781a7f852b761a6e079e5"/>
+ </lrm_resource>
+ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="24:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;24:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="33" rc-code="7" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="547" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
+ </lrm_resource>
+ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="25:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;25:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="37" rc-code="7" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="527" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
+ </lrm_resource>
+ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
+ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="24:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;24:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="3" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
+ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="23:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;23:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="2" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="22:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;22:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-4" call-id="43" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="123" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-0" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-0">
+ <lrm_rsc_op id="stateful-bundle-0_last_0" operation_key="stateful-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="54:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;54:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="10" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="0" queue-time="0" op-digest="bd3b59fd6cf7aabf0f8a6eed8a0e2a71" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-2" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-2">
+ <lrm_rsc_op id="stateful-bundle-2_last_0" operation_key="stateful-bundle-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="158:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;158:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="13" rc-code="0" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="7557a78dd63ce4c65d8c11dedd989a74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="stateful-bundle-2_monitor_30000" operation_key="stateful-bundle-2_monitor_30000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="123:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;123:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="14" rc-code="0" op-status="0" interval="30000" last-rc-change="1592409571" exec-time="0" queue-time="0" op-digest="712ee29a6c182562060ccfbab6326030"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-1" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-1">
+ <lrm_rsc_op id="stateful-bundle-1_last_0" operation_key="stateful-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="57:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;57:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="11" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="98c4879a95dbe1b4a991ce576dd25733" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-0_last_0" operation_key="stateful-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="53:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;53:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="122" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="111" queue-time="0" op-digest="3a787e755377f774b51e7be6dd61684d"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-1_last_0" operation_key="stateful-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="56:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;56:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="130" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="104" queue-time="0" op-digest="7652b221fbab60cc0bb8a0edc188cf31"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="52:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;52:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="118" rc-code="7" op-status="0" interval="0" last-rc-change="1592409564" last-run="1592409564" exec-time="69" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="55:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;55:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="126" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="136" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="154:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;154:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="140" rc-code="0" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="86" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_monitor_60000" operation_key="stateful-bundle-ip-192.168.122.133_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="155:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;155:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="142" rc-code="0" op-status="0" interval="60000" last-rc-change="1592409567" exec-time="158" queue-time="0" op-digest="be05af11daf7c113782f16adb340ab8e"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-2_last_0" operation_key="stateful-bundle-docker-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="156:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;156:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="143" rc-code="0" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="919" queue-time="0" op-digest="3d5c11040df7f5f8b4f7d0ab9675cf2e"/>
+ <lrm_rsc_op id="stateful-bundle-docker-2_monitor_60000" operation_key="stateful-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="157:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;157:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="147" rc-code="0" op-status="0" interval="60000" last-rc-change="1592409568" exec-time="214" queue-time="0" op-digest="bcd89d41e579f1f7f4ff83e715ab91f6"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="4">
+ <instance_attributes id="status-4">
+ <nvpair id="status-4-master-rsc1" name="master-rsc1" value="10"/>
+ <nvpair id="status-4-master-rsc2" name="master-rsc2" value="10"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="3" uname="rhel7-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="32:16:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;32:16:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="40" rc-code="0" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="1" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="20:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;20:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="54" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="10:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;10:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="19" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="46" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="30:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;30:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="21" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="85" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="38:3:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;38:3:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-3" call-id="23" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="24" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="remote-rhel7-2" type="remote" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="remote-rhel7-2_last_0" operation_key="remote-rhel7-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="13:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:7;13:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-3" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1592340505" last-run="1592340505" exec-time="0" queue-time="0" op-digest="f1c162fde9f781a7f852b761a6e079e5" op-force-restart=" server " op-restart-digest="f1c162fde9f781a7f852b761a6e079e5"/>
+ </lrm_resource>
+ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container1_last_0" operation_key="container1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="82:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;82:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="51" rc-code="0" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="1323" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
+ <lrm_rsc_op id="container1_monitor_20000" operation_key="container1_monitor_20000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="90:19:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;90:19:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="133" rc-code="0" op-status="0" interval="20000" last-rc-change="1592412695" exec-time="168" queue-time="0" op-digest="2f32452445eb6ea256d49ee4722bbe31"/>
+ </lrm_resource>
+ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container2_last_0" operation_key="container2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="83:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;83:17:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="59" rc-code="0" op-status="0" interval="0" last-rc-change="1592342393" last-run="1592342393" exec-time="1508" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
+ <lrm_rsc_op id="container2_monitor_20000" operation_key="container2_monitor_20000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="92:23:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;92:23:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="138" rc-code="0" op-status="0" interval="20000" last-rc-change="1592413528" exec-time="158" queue-time="0" op-digest="915ba6aed5781017ce3ca1952a6f0981"/>
+ <lrm_rsc_op id="container2_last_failure_0" operation_key="container2_monitor_20000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="92:23:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;92:23:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="138" rc-code="7" op-status="0" interval="20000" last-rc-change="1592413669" exec-time="0" queue-time="0" op-digest="915ba6aed5781017ce3ca1952a6f0981"/>
+ </lrm_resource>
+ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
+ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="94:18:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;94:18:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="5" rc-code="0" op-status="0" interval="0" last-rc-change="1592342394" last-run="1592342394" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="lxc2_monitor_30000" operation_key="lxc2_monitor_30000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="96:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;96:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="7" rc-code="0" op-status="0" interval="30000" last-rc-change="1592342403" exec-time="0" queue-time="0" op-digest="02a5bcf940fc8d3239701acb11438d6a"/>
+ </lrm_resource>
+ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
+ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="92:18:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;92:18:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="4" rc-code="0" op-status="0" interval="0" last-rc-change="1592342394" last-run="1592342394" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="lxc1_monitor_30000" operation_key="lxc1_monitor_30000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="93:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;93:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="6" rc-code="0" op-status="0" interval="30000" last-rc-change="1592342403" exec-time="0" queue-time="0" op-digest="02a5bcf940fc8d3239701acb11438d6a"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="19:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;19:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="56" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="72" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-0" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-0">
+ <lrm_rsc_op id="stateful-bundle-0_last_0" operation_key="stateful-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="45:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;45:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="11" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="0" queue-time="0" op-digest="bd3b59fd6cf7aabf0f8a6eed8a0e2a71" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-2" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-2">
+ <lrm_rsc_op id="stateful-bundle-2_last_0" operation_key="stateful-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="51:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;51:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="13" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="7557a78dd63ce4c65d8c11dedd989a74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-1" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-1">
+ <lrm_rsc_op id="stateful-bundle-1_last_0" operation_key="stateful-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="48:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;48:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="12" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="98c4879a95dbe1b4a991ce576dd25733" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-0_last_0" operation_key="stateful-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="44:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;44:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="113" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="160" queue-time="0" op-digest="3a787e755377f774b51e7be6dd61684d"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-1_last_0" operation_key="stateful-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="47:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;47:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="121" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="96" queue-time="0" op-digest="7652b221fbab60cc0bb8a0edc188cf31"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="43:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;43:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="109" rc-code="7" op-status="0" interval="0" last-rc-change="1592409564" last-run="1592409564" exec-time="72" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="46:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;46:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="117" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="199" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="49:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;49:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="125" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="141" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-2_last_0" operation_key="stateful-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="50:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;50:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="130" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="77" queue-time="0" op-digest="3d5c11040df7f5f8b4f7d0ab9675cf2e"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-3-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="5" uname="rhel7-5" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="5">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:0:7:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:7;13:0:7:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="10" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="24:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;24:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;13:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="18" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="48" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="34:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;34:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="20" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="35:2:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;35:2:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-5" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="40" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="remote-rhel7-2" type="remote" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="remote-rhel7-2_last_0" operation_key="remote-rhel7-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="15:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:7;15:2:7:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-5" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1592340505" last-run="1592340505" exec-time="0" queue-time="0" op-digest="f1c162fde9f781a7f852b761a6e079e5" op-force-restart=" server " op-restart-digest="f1c162fde9f781a7f852b761a6e079e5"/>
+ </lrm_resource>
+ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="28:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;28:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-5" call-id="35" rc-code="7" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="355" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
+ </lrm_resource>
+ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="29:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;29:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-5" call-id="39" rc-code="7" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="340" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
+ </lrm_resource>
+ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
+ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="27:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;27:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-5" call-id="3" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
+ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="26:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;26:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-5" call-id="2" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="25:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;25:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-5" call-id="44" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="44" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-0" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-0">
+ <lrm_rsc_op id="stateful-bundle-0_last_0" operation_key="stateful-bundle-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="146:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;146:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="11" rc-code="0" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="0" queue-time="0" op-digest="bd3b59fd6cf7aabf0f8a6eed8a0e2a71" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="stateful-bundle-0_monitor_30000" operation_key="stateful-bundle-0_monitor_30000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="109:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;109:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="14" rc-code="0" op-status="0" interval="30000" last-rc-change="1592409570" exec-time="0" queue-time="0" op-digest="039dfa92620d65de0a455ebf46bf1bb6"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-2" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-2">
+ <lrm_rsc_op id="stateful-bundle-2_last_0" operation_key="stateful-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="69:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;69:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="13" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="7557a78dd63ce4c65d8c11dedd989a74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-1" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-1">
+ <lrm_rsc_op id="stateful-bundle-1_last_0" operation_key="stateful-bundle-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="66:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;66:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="12" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="98c4879a95dbe1b4a991ce576dd25733" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-0_last_0" operation_key="stateful-bundle-docker-0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="144:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;144:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="131" rc-code="0" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="1084" queue-time="0" op-digest="3a787e755377f774b51e7be6dd61684d"/>
+ <lrm_rsc_op id="stateful-bundle-docker-0_monitor_60000" operation_key="stateful-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="145:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;145:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="134" rc-code="7" op-status="4" interval="60000" last-rc-change="1592409567" exec-time="300" queue-time="0" op-digest="de3cb764bdefc60ad83989eb8c57a8d0"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-1_last_0" operation_key="stateful-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="65:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;65:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="119" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="107" queue-time="0" op-digest="7652b221fbab60cc0bb8a0edc188cf31"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="142:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;142:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="124" rc-code="0" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="179" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_monitor_60000" operation_key="stateful-bundle-ip-192.168.122.131_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="143:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;143:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="130" rc-code="0" op-status="0" interval="60000" last-rc-change="1592409566" exec-time="180" queue-time="0" op-digest="fc520faa5d1d11c458a6d451e0e08a0e"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="64:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;64:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="115" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="149" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="67:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;67:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="123" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="138" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-2_last_0" operation_key="stateful-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="68:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;68:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="128" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="106" queue-time="0" op-digest="3d5c11040df7f5f8b4f7d0ab9675cf2e"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="5">
+ <instance_attributes id="status-5">
+ <nvpair id="status-5-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-5-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="1" uname="rhel7-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="16:2:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:0;16:2:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-1" call-id="30" rc-code="0" op-status="0" interval="0" last-rc-change="1592340505" last-run="1592340505" exec-time="0" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-0" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-0">
+ <lrm_rsc_op id="stateful-bundle-0_last_0" operation_key="stateful-bundle-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="36:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;36:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="12" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="0" queue-time="0" op-digest="bd3b59fd6cf7aabf0f8a6eed8a0e2a71" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-2" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-2">
+ <lrm_rsc_op id="stateful-bundle-2_last_0" operation_key="stateful-bundle-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="42:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;42:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="15" rc-code="7" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="7557a78dd63ce4c65d8c11dedd989a74" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-1" type="remote" class="ocf" provider="pacemaker" container="stateful-bundle-docker-1">
+ <lrm_rsc_op id="stateful-bundle-1_last_0" operation_key="stateful-bundle-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="152:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;152:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="14" rc-code="0" op-status="0" interval="0" last-rc-change="1592409568" last-run="1592409568" exec-time="0" queue-time="0" op-digest="98c4879a95dbe1b4a991ce576dd25733" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="stateful-bundle-1_monitor_30000" operation_key="stateful-bundle-1_monitor_30000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="116:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;116:3:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="16" rc-code="0" op-status="0" interval="30000" last-rc-change="1592409570" exec-time="0" queue-time="0" op-digest="3ad2e1d8539d1e6893d4a32eaae0c4de"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-0_last_0" operation_key="stateful-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="35:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;35:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="125" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="76" queue-time="0" op-digest="3a787e755377f774b51e7be6dd61684d"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-1_last_0" operation_key="stateful-bundle-docker-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="150:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;150:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="145" rc-code="0" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="1065" queue-time="0" op-digest="7652b221fbab60cc0bb8a0edc188cf31"/>
+ <lrm_rsc_op id="stateful-bundle-docker-1_monitor_60000" operation_key="stateful-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="151:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;151:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="149" rc-code="0" op-status="0" interval="60000" last-rc-change="1592409568" exec-time="377" queue-time="3" op-digest="fcd9e8b06389130ce68ea7bebcc14800"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="34:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;34:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="121" rc-code="7" op-status="0" interval="0" last-rc-change="1592409564" last-run="1592409564" exec-time="80" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="148:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;148:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="142" rc-code="0" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="108" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_monitor_60000" operation_key="stateful-bundle-ip-192.168.122.132_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="149:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;149:2:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="144" rc-code="0" op-status="0" interval="60000" last-rc-change="1592409567" exec-time="157" queue-time="0" op-digest="d35d9d188df006e76b4a9cc497a32049"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="40:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;40:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="137" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="66" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-2_last_0" operation_key="stateful-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="41:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;41:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="141" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="64" queue-time="0" op-digest="3d5c11040df7f5f8b4f7d0ab9675cf2e"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="26:0:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;26:0:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="20" rc-code="0" op-status="0" interval="0" last-rc-change="1592335280" last-run="1592335280" exec-time="88" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="16:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;16:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="45" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="36:1:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;36:1:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="24" rc-code="0" op-status="0" interval="0" last-rc-change="1592335281" last-run="1592335281" exec-time="98" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="38:2:0:89138280-8744-4069-a8c5-69223429df85" transition-magic="0:0;38:2:0:89138280-8744-4069-a8c5-69223429df85" exit-reason="" on_node="rhel7-1" call-id="26" rc-code="0" op-status="0" interval="11000" last-rc-change="1592335281" exec-time="31" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="remote-rhel7-2" type="remote" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="remote-rhel7-2_last_0" operation_key="remote-rhel7-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="58:3:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:0;58:3:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-1" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1592340505" last-run="1592340505" exec-time="0" queue-time="0" op-digest="f1c162fde9f781a7f852b761a6e079e5" op-force-restart=" server " op-restart-digest="f1c162fde9f781a7f852b761a6e079e5"/>
+ <lrm_rsc_op id="remote-rhel7-2_monitor_60000" operation_key="remote-rhel7-2_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="57:4:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:0;57:4:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-1" call-id="3" rc-code="7" op-status="4" interval="60000" last-rc-change="1592340508" exec-time="0" queue-time="0" op-digest="fd28010af096484e2129329ce206b589"/>
+ </lrm_resource>
+ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="16:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;16:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="61" rc-code="7" op-status="0" interval="0" last-rc-change="1592342391" last-run="1592342391" exec-time="178" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
+ </lrm_resource>
+ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="15:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;15:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="65" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="210" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
+ </lrm_resource>
+ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
+ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="18:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;18:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592342393" last-run="1592342393" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
+ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="17:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;17:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="4" rc-code="7" op-status="0" interval="0" last-rc-change="1592342393" last-run="1592342393" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="16:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;16:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="70" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="17" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-1-master-rsc2" name="master-rsc2" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="remote-rhel7-2" uname="remote-rhel7-2" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
+ <lrm id="remote-rhel7-2">
+ <lrm_resources>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="25:4:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:0;25:4:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-1" call-id="13" rc-code="0" op-status="0" interval="0" last-rc-change="1592340508" last-run="1592340508" exec-time="187" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="26:5:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" transition-magic="0:0;26:5:0:8c47b2b9-9363-4906-a816-f8f3c7c82bd2" exit-reason="" on_node="rhel7-1" call-id="29" rc-code="0" op-status="0" interval="11000" last-rc-change="1592340509" exec-time="16" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="45:1:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" transition-magic="0:0;45:1:0:00a37a7a-7ac2-4d2b-b272-e4526634552b" exit-reason="" on_node="rhel7-1" call-id="77" rc-code="0" op-status="0" interval="0" last-rc-change="1592340727" last-run="1592340727" exec-time="184" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_10000" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="47:2:8:00a37a7a-7ac2-4d2b-b272-e4526634552b" transition-magic="0:8;47:2:8:00a37a7a-7ac2-4d2b-b272-e4526634552b" exit-reason="" on_node="rhel7-1" call-id="85" rc-code="8" op-status="0" interval="10000" last-rc-change="1592340727" exec-time="20" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="rsc2_last_failure_0" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="47:2:8:00a37a7a-7ac2-4d2b-b272-e4526634552b" transition-magic="0:7;47:2:8:00a37a7a-7ac2-4d2b-b272-e4526634552b" exit-reason="" on_node="rhel7-1" call-id="85" rc-code="7" op-status="0" interval="10000" last-rc-change="1592340757" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="14:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;14:16:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="49" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="156" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
+ </lrm_resource>
+ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;13:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="53" rc-code="7" op-status="0" interval="0" last-rc-change="1592342392" last-run="1592342392" exec-time="113" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="14:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;14:17:7:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-1" call-id="58" rc-code="7" op-status="0" interval="0" last-rc-change="1592342393" last-run="1592342393" exec-time="13" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="28:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;28:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="341" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="98" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-0_last_0" operation_key="stateful-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="29:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;29:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="345" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="89" queue-time="0" op-digest="3a787e755377f774b51e7be6dd61684d"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="30:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;30:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="349" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="86" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-1_last_0" operation_key="stateful-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="31:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;31:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="353" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="78" queue-time="0" op-digest="7652b221fbab60cc0bb8a0edc188cf31"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="32:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;32:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="357" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="57" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-docker-2_last_0" operation_key="stateful-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="33:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;33:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="361" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="59" queue-time="0" op-digest="3d5c11040df7f5f8b4f7d0ab9675cf2e"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="remote-rhel7-2">
+ <instance_attributes id="status-remote-rhel7-2">
+ <nvpair id="status-remote-rhel7-2-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-remote-rhel7-2-master-rsc2" name="master-rsc2" value="10"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="lxc2" uname="lxc2" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
+ <lrm id="lxc2">
+ <lrm_resources>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="38:19:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;38:19:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1592342401" last-run="1592342401" exec-time="187" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="33:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;33:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592342402" exec-time="36" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="59:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;59:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="23" rc-code="0" op-status="0" interval="0" last-rc-change="1592342402" last-run="1592342402" exec-time="192" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="59:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;59:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="29" rc-code="0" op-status="0" interval="11000" last-rc-change="1592342402" exec-time="45" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="80:22:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;80:22:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="42" rc-code="0" op-status="0" interval="0" last-rc-change="1592342404" last-run="1592342404" exec-time="195" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="lxc-ms_monitor_10000" operation_key="lxc-ms_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="83:24:8:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:8;83:24:8:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="62" rc-code="8" op-status="0" interval="10000" last-rc-change="1592342844" exec-time="33" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="lxc-ms_last_failure_0" operation_key="lxc-ms_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="83:24:8:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:7;83:24:8:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="62" rc-code="7" op-status="0" interval="10000" last-rc-change="1592342864" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="25:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;25:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="87" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="85" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="26:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;26:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="91" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="53" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="27:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;27:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="95" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="96" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="lxc2">
+ <instance_attributes id="status-lxc2">
+ <nvpair id="status-lxc2-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-lxc2-master-rsc2" name="master-rsc2" value="5"/>
+ <nvpair id="status-lxc2-master-lxc-ms" name="master-lxc-ms" value="10"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="lxc1" uname="lxc1" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
+ <lrm id="lxc1">
+ <lrm_resources>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="36:19:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;36:19:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="12" rc-code="0" op-status="0" interval="0" last-rc-change="1592342401" last-run="1592342401" exec-time="236" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="36:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;36:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="18" rc-code="0" op-status="0" interval="11000" last-rc-change="1592342402" exec-time="52" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="57:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;57:20:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="19" rc-code="0" op-status="0" interval="0" last-rc-change="1592342402" last-run="1592342402" exec-time="218" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_11000" operation_key="rsc2_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="62:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;62:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="25" rc-code="0" op-status="0" interval="11000" last-rc-change="1592342402" exec-time="67" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="79:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;79:21:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="36" rc-code="0" op-status="0" interval="0" last-rc-change="1592342403" last-run="1592342403" exec-time="230" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="lxc-ms_monitor_10000" operation_key="lxc-ms_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="83:22:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;83:22:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="42" rc-code="0" op-status="0" interval="10000" last-rc-change="1592342404" exec-time="47" queue-time="0" op-digest="8f6a313464b7f9e3a31cb448458b700e"/>
+ <lrm_rsc_op id="lxc-ms_monitor_11000" operation_key="lxc-ms_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="86:24:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" transition-magic="0:0;86:24:0:23ff9f95-709f-4d37-9fbe-2b07725a45b9" exit-reason="" on_node="rhel7-3" call-id="57" rc-code="0" op-status="0" interval="11000" last-rc-change="1592342844" exec-time="32" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.131" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.131_last_0" operation_key="stateful-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="22:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;22:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="87" rc-code="7" op-status="0" interval="0" last-rc-change="1592409565" last-run="1592409565" exec-time="66" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.132" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.132_last_0" operation_key="stateful-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="23:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;23:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="91" rc-code="7" op-status="0" interval="0" last-rc-change="1592409566" last-run="1592409566" exec-time="53" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-bundle-ip-192.168.122.133" type="IPaddr2" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="stateful-bundle-ip-192.168.122.133_last_0" operation_key="stateful-bundle-ip-192.168.122.133_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="24:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;24:2:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="95" rc-code="7" op-status="0" interval="0" last-rc-change="1592409567" last-run="1592409567" exec-time="61" queue-time="0" op-digest="f318115a675fd430c293a0dc2705f398"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="lxc1">
+ <instance_attributes id="status-lxc1">
+ <nvpair id="status-lxc1-master-rsc1" name="master-rsc1" value="5"/>
+ <nvpair id="status-lxc1-master-rsc2" name="master-rsc2" value="5"/>
+ <nvpair id="status-lxc1-master-lxc-ms" name="master-lxc-ms" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="stateful-bundle-2" uname="stateful-bundle-2" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0">
+ <lrm id="stateful-bundle-2">
+ <lrm_resources>
+ <lrm_resource id="bundled" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="bundled_last_0" operation_key="bundled_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="132:5:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;132:5:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="8" rc-code="0" op-status="0" interval="0" last-rc-change="1592409572" last-run="1592409572" exec-time="241" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="bundled_monitor_11000" operation_key="bundled_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="134:6:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;134:6:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="14" rc-code="0" op-status="0" interval="11000" last-rc-change="1592409572" exec-time="16" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="stateful-bundle-2">
+ <instance_attributes id="status-stateful-bundle-2">
+ <nvpair id="status-stateful-bundle-2-master-bundled" name="master-bundled" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="stateful-bundle-0" uname="stateful-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
+ <lrm id="stateful-bundle-0">
+ <lrm_resources>
+ <lrm_resource id="bundled" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="bundled_last_0" operation_key="bundled_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="128:6:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;128:6:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="14" rc-code="0" op-status="0" interval="0" last-rc-change="1592409572" last-run="1592409572" exec-time="164" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="bundled_monitor_10000" operation_key="bundled_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="131:7:8:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:8;131:7:8:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="20" rc-code="8" op-status="0" interval="10000" last-rc-change="1592409572" exec-time="34" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="bundled_last_failure_0" operation_key="bundled_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="131:7:8:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;131:7:8:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="20" rc-code="7" op-status="0" interval="10000" last-rc-change="1592409773" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="stateful-bundle-0">
+ <instance_attributes id="status-stateful-bundle-0">
+ <nvpair id="status-stateful-bundle-0-master-bundled" name="master-bundled" value="10"/>
+ <nvpair id="status-stateful-bundle-0-fail-count-bundled.monitor_10000" name="fail-count-bundled#monitor_10000" value="1"/>
+ <nvpair id="status-stateful-bundle-0-last-failure-bundled.monitor_10000" name="last-failure-bundled#monitor_10000" value="1592409773"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="stateful-bundle-1" uname="stateful-bundle-1" in_ccm="true" crm-debug-origin="do_state_transition" node_fenced="0">
+ <lrm id="stateful-bundle-1">
+ <lrm_resources>
+ <lrm_resource id="bundled" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="bundled_last_0" operation_key="bundled_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="129:4:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;129:4:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="8" rc-code="0" op-status="0" interval="0" last-rc-change="1592409571" last-run="1592409571" exec-time="225" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="bundled_monitor_11000" operation_key="bundled_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="131:5:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;131:5:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="14" rc-code="0" op-status="0" interval="11000" last-rc-change="1592409572" exec-time="14" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="stateful-bundle-1">
+ <instance_attributes id="status-stateful-bundle-1">
+ <nvpair id="status-stateful-bundle-1-master-bundled" name="master-bundled" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ </status>
+</cib>
--
1.8.3.1
From 204961e95d9de140d998d71a0e53b5b9baa5d39e Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 26 May 2020 18:04:32 -0500
Subject: [PATCH 12/20] Doc: Pacemaker Explained: document new on-fail="demote"
option
---
doc/Pacemaker_Explained/en-US/Ch-Resources.txt | 36 ++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/doc/Pacemaker_Explained/en-US/Ch-Resources.txt b/doc/Pacemaker_Explained/en-US/Ch-Resources.txt
index d8e7115..9df9243 100644
--- a/doc/Pacemaker_Explained/en-US/Ch-Resources.txt
+++ b/doc/Pacemaker_Explained/en-US/Ch-Resources.txt
@@ -676,6 +676,10 @@ a|The action to take if this action ever fails. Allowed values:
* +ignore:+ Pretend the resource did not fail.
* +block:+ Don't perform any further operations on the resource.
* +stop:+ Stop the resource and do not start it elsewhere.
+* +demote:+ Demote the resource, without a full restart. This is valid only for
+ +promote+ actions, and for +monitor+ actions with both a nonzero +interval+
+ and +role+ set to +Master+; for any other action, a configuration error will
+ be logged, and the default behavior will be used.
* +restart:+ Stop the resource and start it again (possibly on a different node).
* +fence:+ STONITH the node on which the resource failed.
* +standby:+ Move _all_ resources away from the node on which the resource failed.
@@ -714,6 +718,38 @@ indexterm:[Action,Property,on-fail]
|=========================================================
+[NOTE]
+====
+When +on-fail+ is set to +demote+, recovery from failure by a successful demote
+causes the cluster to recalculate whether and where a new instance should be
+promoted. The node with the failure is eligible, so if master scores have not
+changed, it will be promoted again.
+
+There is no direct equivalent of +migration-threshold+ for the master role, but
+the same effect can be achieved with a location constraint using a
+<<ch-rules,rule>> with a node attribute expression for the resource's fail
+count.
+
+For example, to immediately ban the master role from a node with any failed
+promote or master monitor:
+[source,XML]
+----
+<rsc_location id="loc1" rsc="my_primitive">
+ <rule id="rule1" score="-INFINITY" role="Master" boolean-op="or">
+ <expression id="expr1" attribute="fail-count-my_primitive#promote_0"
+ operation="gte" value="1"/>
+ <expression id="expr2" attribute="fail-count-my_primitive#monitor_10000"
+ operation="gte" value="1"/>
+ </rule>
+</rsc_location>
+----
+
+This example assumes that there is a promotable clone of the +my_primitive+
+resource (note that the primitive name, not the clone name, is used in the
+rule), and that there is a recurring 10-second-interval monitor configured for
+the master role (fail count attributes specify the interval in milliseconds).
+====
+
[[s-resource-monitoring]]
=== Monitoring Resources for Failure ===
--
1.8.3.1
From d4b9117e72b178bb6f4458cd89bee13060f78dcb Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 26 May 2020 18:10:33 -0500
Subject: [PATCH 13/20] Doc: Pacemaker Explained: correct on-fail default
---
doc/Pacemaker_Explained/en-US/Ch-Resources.txt | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/doc/Pacemaker_Explained/en-US/Ch-Resources.txt b/doc/Pacemaker_Explained/en-US/Ch-Resources.txt
index 9df9243..88892db 100644
--- a/doc/Pacemaker_Explained/en-US/Ch-Resources.txt
+++ b/doc/Pacemaker_Explained/en-US/Ch-Resources.txt
@@ -669,8 +669,13 @@ XML attributes take precedence over +nvpair+ elements if both are specified.
indexterm:[Action,Property,timeout]
|on-fail
-|restart '(except for +stop+ operations, which default to' fence 'when
- STONITH is enabled and' block 'otherwise)'
+a|Varies by action:
+
+* +stop+: +fence+ if +stonith-enabled+ is true or +block+ otherwise
+* +demote+: +on-fail+ of the +monitor+ action with +role+ set to +Master+, if
+ present, enabled, and configured to a value other than +demote+, or +restart+
+ otherwise
+* all other actions: +restart+
a|The action to take if this action ever fails. Allowed values:
* +ignore:+ Pretend the resource did not fail.
--
1.8.3.1
From 0b683445318c783ecef8d6f023b35a6c056ee321 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 2 Jun 2020 15:05:56 -0500
Subject: [PATCH 14/20] Refactor: scheduler: functionize checking quorum policy
in effect
... for readability and ease of future changes
---
lib/pengine/utils.c | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index fee9efb..5d6b836 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -481,6 +481,17 @@ sort_rsc_priority(gconstpointer a, gconstpointer b)
return 0;
}
+static enum pe_quorum_policy
+effective_quorum_policy(pe_resource_t *rsc, pe_working_set_t *data_set)
+{
+ enum pe_quorum_policy policy = data_set->no_quorum_policy;
+
+ if (is_set(data_set->flags, pe_flag_have_quorum)) {
+ policy = no_quorum_ignore;
+ }
+ return policy;
+}
+
pe_action_t *
custom_action(pe_resource_t * rsc, char *key, const char *task,
pe_node_t * on_node, gboolean optional, gboolean save_action,
@@ -593,6 +604,7 @@ custom_action(pe_resource_t * rsc, char *key, const char *task,
if (rsc != NULL) {
enum action_tasks a_task = text2task(action->task);
+ enum pe_quorum_policy quorum_policy = effective_quorum_policy(rsc, data_set);
int warn_level = LOG_TRACE;
if (save_action) {
@@ -675,13 +687,11 @@ custom_action(pe_resource_t * rsc, char *key, const char *task,
crm_trace("Action %s requires only stonith", action->uuid);
action->runnable = TRUE;
#endif
- } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
- && data_set->no_quorum_policy == no_quorum_stop) {
+ } else if (quorum_policy == no_quorum_stop) {
pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "no quorum", pe_action_runnable, TRUE);
crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid);
- } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
- && data_set->no_quorum_policy == no_quorum_freeze) {
+ } else if (quorum_policy == no_quorum_freeze) {
pe_rsc_trace(rsc, "Check resource is already active: %s %s %s %s", rsc->id, action->uuid, role2text(rsc->next_role), role2text(rsc->role));
if (rsc->fns->active(rsc, TRUE) == FALSE || rsc->next_role > rsc->role) {
pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "quorum freeze", pe_action_runnable, TRUE);
--
1.8.3.1
From b1ae359382f15e28e90d9144ca7b1d5f04820c10 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 2 Jun 2020 15:06:32 -0500
Subject: [PATCH 15/20] Feature: scheduler: support "demote" choice for
no-quorum-policy option
If quorum is lost, promotable resources in the master role will be demoted but
left running, and all other resources will be stopped.
---
daemons/controld/controld_control.c | 2 +-
include/crm/pengine/pe_types.h | 3 ++-
lib/common/options.c | 1 +
lib/pengine/common.c | 2 +-
lib/pengine/pe_output.c | 14 ++++++++++++++
lib/pengine/unpack.c | 7 +++++++
lib/pengine/utils.c | 14 ++++++++++++++
7 files changed, 40 insertions(+), 3 deletions(-)
diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c
index 7d29205..059eb7b 100644
--- a/daemons/controld/controld_control.c
+++ b/daemons/controld/controld_control.c
@@ -626,7 +626,7 @@ static pcmk__cluster_option_t crmd_opts[] = {
// Already documented in libpe_status (other values must be kept identical)
{
- "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, suicide",
+ "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, demote, suicide",
"stop", pcmk__valid_quorum, NULL, NULL
},
{
diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h
index ed5eb12..f3cb4ef 100644
--- a/include/crm/pengine/pe_types.h
+++ b/include/crm/pengine/pe_types.h
@@ -61,7 +61,8 @@ enum pe_quorum_policy {
no_quorum_freeze,
no_quorum_stop,
no_quorum_ignore,
- no_quorum_suicide
+ no_quorum_suicide,
+ no_quorum_demote
};
enum node_type {
diff --git a/lib/common/options.c b/lib/common/options.c
index 9399642..9e041c9 100644
--- a/lib/common/options.c
+++ b/lib/common/options.c
@@ -407,6 +407,7 @@ pcmk__valid_quorum(const char *value)
return safe_str_eq(value, "stop")
|| safe_str_eq(value, "freeze")
|| safe_str_eq(value, "ignore")
+ || safe_str_eq(value, "demote")
|| safe_str_eq(value, "suicide");
}
diff --git a/lib/pengine/common.c b/lib/pengine/common.c
index f4f2106..37f287b 100644
--- a/lib/pengine/common.c
+++ b/lib/pengine/common.c
@@ -54,7 +54,7 @@ static pcmk__cluster_option_t pe_opts[] = {
* long description
*/
{
- "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, suicide",
+ "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, demote, suicide",
"stop", pcmk__valid_quorum,
"What to do when the cluster does not have quorum",
NULL
diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
index 75bf0d5..ad469ab 100644
--- a/lib/pengine/pe_output.c
+++ b/lib/pengine/pe_output.c
@@ -729,6 +729,11 @@ pe__cluster_options_html(pcmk__output_t *out, va_list args) {
out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
break;
+ case no_quorum_demote:
+ out->list_item(out, NULL, "No quorum policy: Demote promotable "
+ "resources and stop all other resources");
+ break;
+
case no_quorum_ignore:
out->list_item(out, NULL, "No quorum policy: Ignore");
break;
@@ -785,6 +790,11 @@ pe__cluster_options_text(pcmk__output_t *out, va_list args) {
out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
break;
+ case no_quorum_demote:
+ out->list_item(out, NULL, "No quorum policy: Demote promotable "
+ "resources and stop all other resources");
+ break;
+
case no_quorum_ignore:
out->list_item(out, NULL, "No quorum policy: Ignore");
break;
@@ -817,6 +827,10 @@ pe__cluster_options_xml(pcmk__output_t *out, va_list args) {
xmlSetProp(node, (pcmkXmlStr) "no-quorum-policy", (pcmkXmlStr) "stop");
break;
+ case no_quorum_demote:
+ xmlSetProp(node, (pcmkXmlStr) "no-quorum-policy", (pcmkXmlStr) "demote");
+ break;
+
case no_quorum_ignore:
xmlSetProp(node, (pcmkXmlStr) "no-quorum-policy", (pcmkXmlStr) "ignore");
break;
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index a219805..a480680 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -268,6 +268,9 @@ unpack_config(xmlNode * config, pe_working_set_t * data_set)
} else if (safe_str_eq(value, "freeze")) {
data_set->no_quorum_policy = no_quorum_freeze;
+ } else if (safe_str_eq(value, "demote")) {
+ data_set->no_quorum_policy = no_quorum_demote;
+
} else if (safe_str_eq(value, "suicide")) {
if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
int do_panic = 0;
@@ -297,6 +300,10 @@ unpack_config(xmlNode * config, pe_working_set_t * data_set)
case no_quorum_stop:
crm_debug("On loss of quorum: Stop ALL resources");
break;
+ case no_quorum_demote:
+ crm_debug("On loss of quorum: "
+ "Demote promotable resources and stop other resources");
+ break;
case no_quorum_suicide:
crm_notice("On loss of quorum: Fence all remaining nodes");
break;
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index 5d6b836..f8b631a 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -488,6 +488,20 @@ effective_quorum_policy(pe_resource_t *rsc, pe_working_set_t *data_set)
if (is_set(data_set->flags, pe_flag_have_quorum)) {
policy = no_quorum_ignore;
+
+ } else if (data_set->no_quorum_policy == no_quorum_demote) {
+ switch (rsc->role) {
+ case RSC_ROLE_MASTER:
+ case RSC_ROLE_SLAVE:
+ if (rsc->next_role > RSC_ROLE_SLAVE) {
+ rsc->next_role = RSC_ROLE_SLAVE;
+ }
+ policy = no_quorum_ignore;
+ break;
+ default:
+ policy = no_quorum_stop;
+ break;
+ }
}
return policy;
}
--
1.8.3.1
From 5d809e136f2927259ad570e409e3bbb68f7ce7b4 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Wed, 17 Jun 2020 12:29:50 -0500
Subject: [PATCH 16/20] Test: scheduler: add regression test for
no-quorum-policy="demote"
---
cts/cts-scheduler.in | 1 +
cts/scheduler/no_quorum_demote.dot | 22 ++++
cts/scheduler/no_quorum_demote.exp | 81 ++++++++++++
cts/scheduler/no_quorum_demote.scores | 72 +++++++++++
cts/scheduler/no_quorum_demote.summary | 38 ++++++
cts/scheduler/no_quorum_demote.xml | 224 +++++++++++++++++++++++++++++++++
6 files changed, 438 insertions(+)
create mode 100644 cts/scheduler/no_quorum_demote.dot
create mode 100644 cts/scheduler/no_quorum_demote.exp
create mode 100644 cts/scheduler/no_quorum_demote.scores
create mode 100644 cts/scheduler/no_quorum_demote.summary
create mode 100644 cts/scheduler/no_quorum_demote.xml
diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
index 0e68e73..9e34379 100644
--- a/cts/cts-scheduler.in
+++ b/cts/cts-scheduler.in
@@ -482,6 +482,7 @@ TESTS = [
[ "on_fail_demote2", "Recovery with on-fail=\"demote\" with promotion on different node" ],
[ "on_fail_demote3", "Recovery with on-fail=\"demote\" with no promotion" ],
[ "on_fail_demote4", "Recovery with on-fail=\"demote\" on failed cluster, remote, guest, and bundle nodes" ],
+ [ "no_quorum_demote", "Promotable demotion and primitive stop with no-quorum-policy=\"demote\"" ],
],
[
[ "history-1", "Correctly parse stateful-1 resource state" ],
diff --git a/cts/scheduler/no_quorum_demote.dot b/cts/scheduler/no_quorum_demote.dot
new file mode 100644
index 0000000..ea5b30c
--- /dev/null
+++ b/cts/scheduler/no_quorum_demote.dot
@@ -0,0 +1,22 @@
+ digraph "g" {
+"Cancel rsc1_monitor_10000 rhel7-1" -> "rsc1_demote_0 rhel7-1" [ style = bold]
+"Cancel rsc1_monitor_10000 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"Fencing_monitor_120000 rhel7-1" [ style=dashed color="red" fontcolor="black"]
+"Fencing_start_0 rhel7-1" -> "Fencing_monitor_120000 rhel7-1" [ style = dashed]
+"Fencing_start_0 rhel7-1" [ style=dashed color="red" fontcolor="black"]
+"Fencing_stop_0 rhel7-1" -> "Fencing_start_0 rhel7-1" [ style = dashed]
+"Fencing_stop_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-1" [ style = bold]
+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1_demote_0 rhel7-1" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1_demote_0 rhel7-1" -> "rsc1_monitor_11000 rhel7-1" [ style = bold]
+"rsc1_demote_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"rsc1_monitor_11000 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"rsc2_monitor_10000 rhel7-2" [ style=dashed color="red" fontcolor="black"]
+"rsc2_start_0 rhel7-2" -> "rsc2_monitor_10000 rhel7-2" [ style = dashed]
+"rsc2_start_0 rhel7-2" [ style=dashed color="red" fontcolor="black"]
+"rsc2_stop_0 rhel7-2" -> "rsc2_start_0 rhel7-2" [ style = dashed]
+"rsc2_stop_0 rhel7-2" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/cts/scheduler/no_quorum_demote.exp b/cts/scheduler/no_quorum_demote.exp
new file mode 100644
index 0000000..245574c
--- /dev/null
+++ b/cts/scheduler/no_quorum_demote.exp
@@ -0,0 +1,81 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="1" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="5" operation="stop" operation_key="Fencing_stop_0" on_node="rhel7-1" on_node_uuid="1">
+ <primitive id="Fencing" class="stonith" type="fence_xvm"/>
+ <attributes CRM_meta_name="stop" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="60000" key_file="/etc/pacemaker/fence_xvm.key" multicast_address="239.255.100.100" pcmk_host_list="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 rhel7-3 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5" pcmk_host_map="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="10" operation="monitor" operation_key="rsc1_monitor_11000" internal_operation_key="rsc1:0_monitor_11000" on_node="rhel7-1" on_node_uuid="1">
+ <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="11000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Slave" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="8" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-1" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="8" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-1" on_node_uuid="1">
+ <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="10000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="2" operation="cancel" operation_key="rsc1_monitor_10000" internal_operation_key="rsc1:0_monitor_10000" on_node="rhel7-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="19" operation="demote" operation_key="rsc1-clone_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="2" operation="cancel" operation_key="rsc1_monitor_10000" internal_operation_key="rsc1:0_monitor_10000" on_node="rhel7-1" on_node_uuid="1">
+ <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_fail="demote" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_operation="monitor" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_role="Master" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+ <pseudo_event id="20" operation="demoted" operation_key="rsc1-clone_demoted_0">
+ <attributes CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="8" operation="demote" operation_key="rsc1_demote_0" internal_operation_key="rsc1:0_demote_0" on_node="rhel7-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="19" operation="demote" operation_key="rsc1-clone_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <pseudo_event id="19" operation="demote" operation_key="rsc1-clone_demote_0">
+ <attributes CRM_meta_clone_max="5" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="90000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <rsc_op id="21" operation="stop" operation_key="rsc2_stop_0" on_node="rhel7-2" on_node_uuid="2">
+ <primitive id="rsc2" class="ocf" provider="pacemaker" type="Dummy"/>
+ <attributes CRM_meta_name="stop" CRM_meta_on_node="rhel7-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+</transition_graph>
diff --git a/cts/scheduler/no_quorum_demote.scores b/cts/scheduler/no_quorum_demote.scores
new file mode 100644
index 0000000..dddc57b
--- /dev/null
+++ b/cts/scheduler/no_quorum_demote.scores
@@ -0,0 +1,72 @@
+Allocation scores:
+Using the original execution date of: 2020-06-17 17:26:35Z
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 11
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-2: 6
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 10
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-2: 5
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 10
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-2: 5
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 10
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-2: 5
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-2: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: 11
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-2: 6
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-3: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-5: 0
+rsc1:0 promotion score on rhel7-1: 10
+rsc1:1 promotion score on rhel7-2: 5
+rsc1:2 promotion score on none: 0
+rsc1:3 promotion score on none: 0
+rsc1:4 promotion score on none: 0
diff --git a/cts/scheduler/no_quorum_demote.summary b/cts/scheduler/no_quorum_demote.summary
new file mode 100644
index 0000000..9b69ca1
--- /dev/null
+++ b/cts/scheduler/no_quorum_demote.summary
@@ -0,0 +1,38 @@
+Using the original execution date of: 2020-06-17 17:26:35Z
+
+Current cluster status:
+Online: [ rhel7-1 rhel7-2 ]
+OFFLINE: [ rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-1
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Masters: [ rhel7-1 ]
+ Slaves: [ rhel7-2 ]
+ Stopped: [ rhel7-3 rhel7-4 rhel7-5 ]
+ rsc2 (ocf::pacemaker:Dummy): Started rhel7-2
+
+Transition Summary:
+ * Stop Fencing ( rhel7-1 ) due to no quorum
+ * Demote rsc1:0 ( Master -> Slave rhel7-1 )
+ * Stop rsc2 ( rhel7-2 ) due to no quorum
+
+Executing cluster transition:
+ * Resource action: Fencing stop on rhel7-1
+ * Resource action: rsc1 cancel=10000 on rhel7-1
+ * Pseudo action: rsc1-clone_demote_0
+ * Resource action: rsc2 stop on rhel7-2
+ * Resource action: rsc1 demote on rhel7-1
+ * Pseudo action: rsc1-clone_demoted_0
+ * Resource action: rsc1 monitor=11000 on rhel7-1
+Using the original execution date of: 2020-06-17 17:26:35Z
+
+Revised cluster status:
+Online: [ rhel7-1 rhel7-2 ]
+OFFLINE: [ rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Stopped
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Slaves: [ rhel7-1 rhel7-2 ]
+ Stopped: [ rhel7-3 rhel7-4 rhel7-5 ]
+ rsc2 (ocf::pacemaker:Dummy): Stopped
+
diff --git a/cts/scheduler/no_quorum_demote.xml b/cts/scheduler/no_quorum_demote.xml
new file mode 100644
index 0000000..8497f0a
--- /dev/null
+++ b/cts/scheduler/no_quorum_demote.xml
@@ -0,0 +1,224 @@
+<cib crm_feature_set="3.4.0" validate-with="pacemaker-3.4" epoch="104" num_updates="50" admin_epoch="1" cib-last-written="Wed Jun 17 12:25:43 2020" update-origin="rhel7-2" update-client="cibadmin" update-user="root" have-quorum="0" dc-uuid="1" no-quorum-panic="1" execution-date="1592414795">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/>
+ <nvpair id="cts-start-failure-is-fatal" name="start-failure-is-fatal" value="false"/>
+ <nvpair id="cts-pe-input-series-max" name="pe-input-series-max" value="5000"/>
+ <nvpair id="cts-shutdown-escalation" name="shutdown-escalation" value="5min"/>
+ <nvpair id="cts-batch-limit" name="batch-limit" value="10"/>
+ <nvpair id="cts-dc-deadtime" name="dc-deadtime" value="5s"/>
+ <nvpair id="cts-no-quorum-policy" name="no-quorum-policy" value="demote"/>
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-579.76fa32b.git.el7_8-76fa32b"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="mycluster"/>
+ <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1591654576"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="rhel7-1">
+ <instance_attributes id="rhel7-1-1">
+ <nvpair id="rhel7-1-1-cts-fencing" name="cts-fencing" value="levels-and"/>
+ </instance_attributes>
+ </node>
+ <node id="2" uname="rhel7-2">
+ <instance_attributes id="rhel7-2-1">
+ <nvpair id="rhel7-2-1-cts-fencing" name="cts-fencing" value="levels-and"/>
+ </instance_attributes>
+ </node>
+ <node id="3" uname="rhel7-3"/>
+ <node id="4" uname="rhel7-4">
+ <instance_attributes id="nodes-4">
+ <nvpair id="nodes-4-standby" name="standby" value="off"/>
+ </instance_attributes>
+ </node>
+ <node id="5" uname="rhel7-5"/>
+ </nodes>
+ <resources>
+ <primitive class="stonith" id="Fencing" type="fence_xvm">
+ <meta_attributes id="Fencing-meta"/>
+ <instance_attributes id="Fencing-params">
+ <nvpair id="Fencing-key_file" name="key_file" value="/etc/pacemaker/fence_xvm.key"/>
+ <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/>
+ <nvpair id="Fencing-pcmk_host_map" name="pcmk_host_map" value="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
+ <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 rhel7-3 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5"/>
+ </instance_attributes>
+ <operations>
+ <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/>
+ <op id="Fencing-stop-0" interval="0" name="stop" timeout="60s"/>
+ <op id="Fencing-start-0" interval="0" name="start" timeout="60s"/>
+ </operations>
+ </primitive>
+ <clone id="rsc1-clone">
+ <meta_attributes id="rsc1-meta_attributes">
+ <nvpair id="rsc1-meta_attributes-promotable" name="promotable" value="true"/>
+ </meta_attributes>
+ <primitive class="ocf" id="rsc1" provider="pacemaker" type="Stateful">
+ <operations>
+ <op id="rsc1-demote-interval-0s" interval="0s" name="demote" timeout="10s"/>
+ <op id="rsc1-monitor-interval-10s" interval="10s" name="monitor" on-fail="demote" role="Master" timeout="20s"/>
+ <op id="rsc1-monitor-interval-11s" interval="11s" name="monitor" role="Slave" timeout="20s"/>
+ <op id="rsc1-notify-interval-0s" interval="0s" name="notify" timeout="5s"/>
+ <op id="rsc1-promote-interval-0s" interval="0s" name="promote" timeout="10s"/>
+ <op id="rsc1-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+ <op id="rsc1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+ </operations>
+ </primitive>
+ </clone>
+ <primitive class="ocf" id="rsc2" provider="pacemaker" type="Dummy">
+ <operations>
+ <op id="rsc2-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+ <op id="rsc2-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+ <op id="rsc2-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
+ <op id="rsc2-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+ <op id="rsc2-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+ <op id="rsc2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+ </operations>
+ </primitive>
+ </resources>
+ <constraints/>
+ <op_defaults>
+ <meta_attributes id="cts-op_defaults-meta">
+ <nvpair id="cts-op_defaults-timeout" name="timeout" value="90s"/>
+ </meta_attributes>
+ </op_defaults>
+ <alerts>
+ <alert id="alert-1" path="/var/lib/pacemaker/notify.sh">
+ <recipient id="alert-1-recipient-1" value="/run/crm/alert.log"/>
+ </alert>
+ </alerts>
+ </configuration>
+ <status>
+ <node_state id="1" uname="rhel7-1" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" join="member" expected="member">
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-master-rsc1" name="master-rsc1" value="10"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="9:86:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;9:86:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="242" rc-code="0" op-status="0" interval="0" last-rc-change="1592414787" last-run="1592414787" exec-time="72" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="7:89:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;7:89:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="245" rc-code="0" op-status="0" interval="120000" last-rc-change="1592414788" exec-time="66" queue-time="0" op-digest="cb34bc19df153021ce8f301baa293f35"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="6:83:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;6:83:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="227" rc-code="0" op-status="0" interval="0" last-rc-change="1592414729" last-run="1592414729" exec-time="66" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_10000" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="12:84:8:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:8;12:84:8:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="229" rc-code="8" op-status="0" interval="10000" last-rc-change="1592414729" exec-time="20" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="30:86:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;30:86:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="240" rc-code="0" op-status="0" interval="0" last-rc-change="1592414787" last-run="1592414787" exec-time="102" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_10000" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="35:85:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;35:85:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="237" rc-code="0" op-status="0" interval="10000" last-rc-change="1592414743" exec-time="19" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="3" uname="rhel7-3" in_ccm="false" crmd="offline" crm-debug-origin="post_cache_update" join="down" expected="down">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="17:0:7:42f7e002-ff77-40f5-81ff-081d5f1fa740" transition-magic="0:7;17:0:7:42f7e002-ff77-40f5-81ff-081d5f1fa740" exit-reason="" on_node="rhel7-3" call-id="9" rc-code="7" op-status="0" interval="0" last-rc-change="1592408637" last-run="1592408637" exec-time="9" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="12:89:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;12:89:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="230" rc-code="0" op-status="0" interval="0" last-rc-change="1592414788" last-run="1592414788" exec-time="94" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="10:83:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;10:83:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="223" rc-code="0" op-status="0" interval="11000" last-rc-change="1592414729" exec-time="53" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="9:85:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;9:85:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-3" call-id="228" rc-code="7" op-status="0" interval="0" last-rc-change="1592414743" last-run="1592414743" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="4" uname="rhel7-4" in_ccm="false" crmd="offline" crm-debug-origin="post_cache_update" join="down" expected="down">
+ <lrm id="4">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="8:86:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;8:86:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="202" rc-code="0" op-status="0" interval="0" last-rc-change="1592414787" last-run="1592414787" exec-time="0" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="48:0:0:42f7e002-ff77-40f5-81ff-081d5f1fa740" transition-magic="0:0;48:0:0:42f7e002-ff77-40f5-81ff-081d5f1fa740" exit-reason="" on_node="rhel7-4" call-id="40" rc-code="0" op-status="0" interval="120000" last-rc-change="1592408639" exec-time="45" queue-time="0" op-digest="cb34bc19df153021ce8f301baa293f35"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="17:86:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;17:86:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="206" rc-code="0" op-status="0" interval="0" last-rc-change="1592414787" last-run="1592414787" exec-time="60" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:83:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;13:83:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="195" rc-code="0" op-status="0" interval="11000" last-rc-change="1592414729" exec-time="30" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="10:85:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;10:85:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="200" rc-code="7" op-status="0" interval="0" last-rc-change="1592414743" last-run="1592414743" exec-time="51" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="5" uname="rhel7-5" in_ccm="false" crmd="offline" crm-debug-origin="post_cache_update" join="down" expected="down">
+ <lrm id="5">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="37:0:7:42f7e002-ff77-40f5-81ff-081d5f1fa740" transition-magic="0:7;37:0:7:42f7e002-ff77-40f5-81ff-081d5f1fa740" exit-reason="" on_node="rhel7-5" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592408637" last-run="1592408637" exec-time="2" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="13:89:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;13:89:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="208" rc-code="0" op-status="0" interval="0" last-rc-change="1592414788" last-run="1592414788" exec-time="111" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="16:83:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;16:83:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="201" rc-code="0" op-status="0" interval="11000" last-rc-change="1592414729" exec-time="38" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="11:85:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;11:85:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="206" rc-code="7" op-status="0" interval="0" last-rc-change="1592414743" last-run="1592414743" exec-time="31" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="5">
+ <instance_attributes id="status-5">
+ <nvpair id="status-5-shutdown" name="shutdown" value="1592414787"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="2" uname="rhel7-2" crmd="online" crm-debug-origin="post_cache_update" in_ccm="true" join="member" expected="member">
+ <transient_attributes id="2">
+ <instance_attributes id="status-2">
+ <nvpair id="status-2-master-rsc1" name="master-rsc1" value="5"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.4.0" transition-key="10:67:7:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;10:67:7:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-2" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1592414636" last-run="1592414636" exec-time="4" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
+ </lrm_resource>
+ <lrm_resource id="rsc1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="11:82:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;11:82:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-2" call-id="39" rc-code="0" op-status="0" interval="0" last-rc-change="1592414729" last-run="1592414729" exec-time="44" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc1_monitor_11000" operation_key="rsc1_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="19:83:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;19:83:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-2" call-id="41" rc-code="0" op-status="0" interval="11000" last-rc-change="1592414729" exec-time="36" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="24:89:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;24:89:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-2" call-id="47" rc-code="0" op-status="0" interval="0" last-rc-change="1592414788" last-run="1592414788" exec-time="30" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="rsc2_monitor_10000" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="25:89:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;25:89:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-2" call-id="49" rc-code="0" op-status="0" interval="10000" last-rc-change="1592414788" exec-time="56" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state remote_node="true" id="stateful-bundle-2" uname="stateful-bundle-2" in_ccm="false" crm-debug-origin="post_cache_update" node_fenced="0">
+ <lrm id="stateful-bundle-2">
+ <lrm_resources>
+ <lrm_resource id="bundled" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="bundled_last_0" operation_key="bundled_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="86:48:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;86:48:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="173" rc-code="0" op-status="0" interval="0" last-rc-change="1592414312" last-run="1592414312" exec-time="208" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="bundled_monitor_11000" operation_key="bundled_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="134:6:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;134:6:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-4" call-id="14" rc-code="0" op-status="0" interval="11000" last-rc-change="1592409572" exec-time="16" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state remote_node="true" id="stateful-bundle-0" uname="stateful-bundle-0" in_ccm="false" crm-debug-origin="post_cache_update" node_fenced="0">
+ <lrm id="stateful-bundle-0">
+ <lrm_resources>
+ <lrm_resource id="bundled" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="bundled_last_0" operation_key="bundled_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="21:50:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;21:50:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="209" rc-code="0" op-status="0" interval="0" last-rc-change="1592414392" last-run="1592414392" exec-time="182" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="bundled_monitor_10000" operation_key="bundled_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="131:15:8:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:8;131:15:8:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="47" rc-code="8" op-status="0" interval="10000" last-rc-change="1592409774" exec-time="14" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="bundled_last_failure_0" operation_key="bundled_demote_0" operation="demote" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="33:11:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:7;33:11:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-5" call-id="28" rc-code="7" op-status="0" interval="0" last-rc-change="1592409773" exec-time="50" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" last-run="1592409773"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state remote_node="true" id="stateful-bundle-1" uname="stateful-bundle-1" in_ccm="false" crm-debug-origin="post_cache_update" node_fenced="0">
+ <lrm id="stateful-bundle-1">
+ <lrm_resources>
+ <lrm_resource id="bundled" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="bundled_last_0" operation_key="bundled_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="84:49:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;84:49:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="175" rc-code="0" op-status="0" interval="0" last-rc-change="1592414312" last-run="1592414312" exec-time="222" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="bundled_monitor_11000" operation_key="bundled_monitor_11000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.4.0" transition-key="132:14:0:00b1bce3-139f-4974-abc7-a01a2248b512" transition-magic="0:0;132:14:0:00b1bce3-139f-4974-abc7-a01a2248b512" exit-reason="" on_node="rhel7-1" call-id="22" rc-code="0" op-status="0" interval="11000" last-rc-change="1592409774" exec-time="23" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ </status>
+</cib>
--
1.8.3.1
From 015b5c012ce41a8035260522f67127135937baa2 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Thu, 28 May 2020 12:13:20 -0500
Subject: [PATCH 17/20] Doc: Pacemaker Explained: document
no-quorum-policy=demote
---
doc/Pacemaker_Explained/en-US/Ch-Options.txt | 2 ++
1 file changed, 2 insertions(+)
diff --git a/doc/Pacemaker_Explained/en-US/Ch-Options.txt b/doc/Pacemaker_Explained/en-US/Ch-Options.txt
index faefe7c..b158f00 100644
--- a/doc/Pacemaker_Explained/en-US/Ch-Options.txt
+++ b/doc/Pacemaker_Explained/en-US/Ch-Options.txt
@@ -181,6 +181,8 @@ What to do when the cluster does not have quorum. Allowed values:
* +ignore:+ continue all resource management
* +freeze:+ continue resource management, but don't recover resources from nodes not in the affected partition
* +stop:+ stop all resources in the affected cluster partition
+* +demote:+ demote promotable resources and stop all other resources in the
+ affected cluster partition
* +suicide:+ fence all nodes in the affected cluster partition
| batch-limit | 0 |
--
1.8.3.1
From 01c5ec67e0a6ee1395d771f8fbaf619a44ab2ca2 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 2 Jun 2020 19:23:11 -0500
Subject: [PATCH 18/20] Low: scheduler: match initial no-quorum-policy struct
value to actual default
It doesn't matter in practice since the actual default is parsed from the
option definition via pe_pref(), but it's confusing to have them different.
---
lib/pengine/status.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/pengine/status.c b/lib/pengine/status.c
index 8dc5095..ca34639 100644
--- a/lib/pengine/status.c
+++ b/lib/pengine/status.c
@@ -360,7 +360,7 @@ set_working_set_defaults(pe_working_set_t * data_set)
data_set->order_id = 1;
data_set->action_id = 1;
- data_set->no_quorum_policy = no_quorum_freeze;
+ data_set->no_quorum_policy = no_quorum_stop;
data_set->flags = 0x0ULL;
set_bit(data_set->flags, pe_flag_stop_rsc_orphans);
--
1.8.3.1
From 7eec572dbba3ade059e5206a2ba496f9da3a68bc Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Fri, 5 Jun 2020 10:02:05 -0500
Subject: [PATCH 19/20] Build: libcrmcommon: bump CRM feature set
... for op_expression/rsc_expression rules, on-fail=demote, and
no-quorum-policy=demote
---
include/crm/crm.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/include/crm/crm.h b/include/crm/crm.h
index d2ffb61..dc2adc1 100644
--- a/include/crm/crm.h
+++ b/include/crm/crm.h
@@ -51,7 +51,7 @@ extern "C" {
* >=3.0.13: Fail counts include operation name and interval
* >=3.2.0: DC supports PCMK_LRM_OP_INVALID and PCMK_LRM_OP_NOT_CONNECTED
*/
-# define CRM_FEATURE_SET "3.3.0"
+# define CRM_FEATURE_SET "3.4.0"
# define EOS '\0'
# define DIMOF(a) ((int) (sizeof(a)/sizeof(a[0])) )
--
1.8.3.1
From c4429d86ef00bb1749adc476f9c6874e3f5d95b9 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 16 Jun 2020 14:38:35 -0500
Subject: [PATCH 20/20] Log: scheduler: downgrade "active on" messages to trace
... now that they're logged more often via pcmk__rsc_is_filtered()
---
lib/pengine/native.c | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/lib/pengine/native.c b/lib/pengine/native.c
index f0d83d7..20658a0 100644
--- a/lib/pengine/native.c
+++ b/lib/pengine/native.c
@@ -359,22 +359,22 @@ native_parameter(pe_resource_t * rsc, pe_node_t * node, gboolean create, const c
gboolean
native_active(pe_resource_t * rsc, gboolean all)
{
- GListPtr gIter = rsc->running_on;
-
- for (; gIter != NULL; gIter = gIter->next) {
+ for (GList *gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
pe_node_t *a_node = (pe_node_t *) gIter->data;
if (a_node->details->unclean) {
- crm_debug("Resource %s: node %s is unclean", rsc->id, a_node->details->uname);
+ pe_rsc_trace(rsc, "Resource %s: node %s is unclean",
+ rsc->id, a_node->details->uname);
return TRUE;
} else if (a_node->details->online == FALSE) {
- crm_debug("Resource %s: node %s is offline", rsc->id, a_node->details->uname);
+ pe_rsc_trace(rsc, "Resource %s: node %s is offline",
+ rsc->id, a_node->details->uname);
} else {
- crm_debug("Resource %s active on %s", rsc->id, a_node->details->uname);
+ pe_rsc_trace(rsc, "Resource %s active on %s",
+ rsc->id, a_node->details->uname);
return TRUE;
}
}
-
return FALSE;
}
--
1.8.3.1