diff --git a/SOURCES/008-bundle-ordering.patch b/SOURCES/008-bundle-ordering.patch
new file mode 100644
index 0000000..916c590
--- /dev/null
+++ b/SOURCES/008-bundle-ordering.patch
@@ -0,0 +1,2386 @@
+From a07ff469d96312b37f9c0c5ac65e7c1f87394ce5 Mon Sep 17 00:00:00 2001
+From: Andrew Beekhof <andrew@beekhof.net>
+Date: Fri, 12 Oct 2018 12:13:14 +1100
+Subject: [PATCH 1/9] Fix: schedulerd: Improve internal bundle ordering
+
+If the remote resource is scheduled to stop, we should at least wait to
+know the state of the underlying container resource before executing it.
+
+Otherwise we may end up needlessly tearing it down just because someone
+ran a cleanup on the container.
+---
+ pengine/container.c | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/pengine/container.c b/pengine/container.c
+index 15d094d..4e77545 100644
+--- a/pengine/container.c
++++ b/pengine/container.c
+@@ -278,6 +278,7 @@ container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
+ order_start_start(rsc, tuple->docker, pe_order_runnable_left | pe_order_implies_first_printed);
+
+ if(tuple->child) {
++ new_rsc_order(tuple->docker, RSC_STATUS, tuple->remote, RSC_STOP, pe_order_optional, data_set);
+ order_stop_stop(rsc, tuple->child, pe_order_implies_first_printed);
+ }
+ order_stop_stop(rsc, tuple->docker, pe_order_implies_first_printed);
+--
+1.8.3.1
+
+
+From 35dc265c4577b7fbe46c8e5919b7ef59028c1de5 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Mon, 29 Oct 2018 14:23:06 -0500
+Subject: [PATCH 2/9] Test: pengine: Improve internal bundle ordering
+
+---
+ pengine/test10/bundle-nested-colocation.exp | 12 +++---
+ pengine/test10/bundle-order-fencing.exp | 56 ++++++++++++-------------
+ pengine/test10/bundle-order-partial-start-2.exp | 12 +++---
+ pengine/test10/bundle-order-partial-start.exp | 12 +++---
+ pengine/test10/bundle-order-partial-stop.exp | 16 +++----
+ pengine/test10/bundle-order-stop-clone.exp | 20 ++++-----
+ pengine/test10/bundle-order-stop-on-remote.exp | 44 +++++++++----------
+ pengine/test10/bundle-order-stop.exp | 16 +++----
+ 8 files changed, 94 insertions(+), 94 deletions(-)
+
+diff --git a/pengine/test10/bundle-nested-colocation.exp b/pengine/test10/bundle-nested-colocation.exp
+index a50809c..29c2eda 100644
+--- a/pengine/test10/bundle-nested-colocation.exp
++++ b/pengine/test10/bundle-nested-colocation.exp
+@@ -1,7 +1,7 @@
+ <transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0" priority="1000000">
+ <action_set>
+- <rsc_op id="83" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="overcloud-controller-0">
++ <rsc_op id="86" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="overcloud-controller-0">
+ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 overcloud-galera-0 overcloud-galera-1 overcloud-galera-2 overcloud-rabbit-0 overcloud-rabbit-1 overcloud-rabbit-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="rabbitmq-bundle-0 overcloud-galera-2 overcloud-galera-1 overcloud-galera-0 overcloud-controller-2 overcloud-controller-1 overcloud-controller-0 overcloud-rabbit-2 overcloud-rabbit-1 overcloud-rabbit-0 rabbitmq-bundle-2 rabbitmq-bundle-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_start_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="overcloud-rabbit-0" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
+ </rsc_op>
+@@ -65,7 +65,7 @@
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+- <rsc_op id="84" operation="notify" operation_key="rabbitmq:1_post_notify_start_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="overcloud-controller-1">
++ <rsc_op id="87" operation="notify" operation_key="rabbitmq:1_post_notify_start_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="overcloud-controller-1">
+ <primitive id="rabbitmq" long-id="rabbitmq:1" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 overcloud-galera-0 overcloud-galera-1 overcloud-galera-2 overcloud-rabbit-0 overcloud-rabbit-1 overcloud-rabbit-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="rabbitmq-bundle-0 overcloud-galera-2 overcloud-galera-1 overcloud-galera-0 overcloud-controller-2 overcloud-controller-1 overcloud-controller-0 overcloud-rabbit-2 overcloud-rabbit-1 overcloud-rabbit-0 rabbitmq-bundle-2 rabbitmq-bundle-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_start_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="rabbitmq-bundle-1" CRM_meta_on_node_uuid="rabbitmq-bundle-1" CRM_meta_physical_host="overcloud-rabbit-1" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
+ </rsc_op>
+@@ -132,7 +132,7 @@
+ </synapse>
+ <synapse id="8" priority="1000000">
+ <action_set>
+- <rsc_op id="85" operation="notify" operation_key="rabbitmq:2_post_notify_start_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="overcloud-controller-2">
++ <rsc_op id="88" operation="notify" operation_key="rabbitmq:2_post_notify_start_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="overcloud-controller-2">
+ <primitive id="rabbitmq" long-id="rabbitmq:2" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 overcloud-galera-0 overcloud-galera-1 overcloud-galera-2 overcloud-rabbit-0 overcloud-rabbit-1 overcloud-rabbit-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="rabbitmq-bundle-0 overcloud-galera-2 overcloud-galera-1 overcloud-galera-0 overcloud-controller-2 overcloud-controller-1 overcloud-controller-0 overcloud-rabbit-2 overcloud-rabbit-1 overcloud-rabbit-0 rabbitmq-bundle-2 rabbitmq-bundle-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_start_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="rabbitmq-bundle-2" CRM_meta_on_node_uuid="rabbitmq-bundle-2" CRM_meta_physical_host="overcloud-rabbit-2" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
+ </rsc_op>
+@@ -208,13 +208,13 @@
+ <pseudo_event id="64" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_running_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="83" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="overcloud-controller-0"/>
++ <rsc_op id="86" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="overcloud-controller-0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="84" operation="notify" operation_key="rabbitmq:1_post_notify_start_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="overcloud-controller-1"/>
++ <rsc_op id="87" operation="notify" operation_key="rabbitmq:1_post_notify_start_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="overcloud-controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="85" operation="notify" operation_key="rabbitmq:2_post_notify_start_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="overcloud-controller-2"/>
++ <rsc_op id="88" operation="notify" operation_key="rabbitmq:2_post_notify_start_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="overcloud-controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+diff --git a/pengine/test10/bundle-order-fencing.exp b/pengine/test10/bundle-order-fencing.exp
+index 599c299..2b8f5cf 100644
+--- a/pengine/test10/bundle-order-fencing.exp
++++ b/pengine/test10/bundle-order-fencing.exp
+@@ -46,7 +46,7 @@
+ </synapse>
+ <synapse id="3">
+ <action_set>
+- <rsc_op id="226" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:1_pre_notify_stop_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="controller-1">
++ <rsc_op id="235" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:1_pre_notify_stop_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="controller-1">
+ <primitive id="rabbitmq" long-id="rabbitmq:1" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="rabbitmq-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="rabbitmq-bundle-1" CRM_meta_on_node_uuid="rabbitmq-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
+ </rsc_op>
+@@ -75,7 +75,7 @@
+ </synapse>
+ <synapse id="5">
+ <action_set>
+- <rsc_op id="227" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:2_pre_notify_stop_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="controller-2">
++ <rsc_op id="236" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:2_pre_notify_stop_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="controller-2">
+ <primitive id="rabbitmq" long-id="rabbitmq:2" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="rabbitmq-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="rabbitmq-bundle-2" CRM_meta_on_node_uuid="rabbitmq-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
+ </rsc_op>
+@@ -146,10 +146,10 @@
+ <pseudo_event id="73" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="226" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:1_pre_notify_stop_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="controller-1"/>
++ <rsc_op id="235" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:1_pre_notify_stop_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="227" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:2_pre_notify_stop_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="controller-2"/>
++ <rsc_op id="236" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:2_pre_notify_stop_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -521,7 +521,7 @@
+ </synapse>
+ <synapse id="39" priority="1000000">
+ <action_set>
+- <rsc_op id="231" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1">
++ <rsc_op id="240" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -534,7 +534,7 @@
+ </synapse>
+ <synapse id="40">
+ <action_set>
+- <rsc_op id="230" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1">
++ <rsc_op id="239" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -607,7 +607,7 @@
+ </synapse>
+ <synapse id="45" priority="1000000">
+ <action_set>
+- <rsc_op id="237" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="246" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -620,7 +620,7 @@
+ </synapse>
+ <synapse id="46">
+ <action_set>
+- <rsc_op id="236" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="245" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -633,7 +633,7 @@
+ </synapse>
+ <synapse id="47" priority="1000000">
+ <action_set>
+- <rsc_op id="233" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="242" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -646,7 +646,7 @@
+ </synapse>
+ <synapse id="48">
+ <action_set>
+- <rsc_op id="232" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="241" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -659,7 +659,7 @@
+ </synapse>
+ <synapse id="49">
+ <action_set>
+- <rsc_op id="228" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:1_pre_notify_stop_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="237" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:1_pre_notify_stop_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -753,7 +753,7 @@
+ </synapse>
+ <synapse id="55" priority="1000000">
+ <action_set>
+- <rsc_op id="239" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="248" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -766,7 +766,7 @@
+ </synapse>
+ <synapse id="56">
+ <action_set>
+- <rsc_op id="238" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="247" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -779,7 +779,7 @@
+ </synapse>
+ <synapse id="57" priority="1000000">
+ <action_set>
+- <rsc_op id="235" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="244" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -792,7 +792,7 @@
+ </synapse>
+ <synapse id="58">
+ <action_set>
+- <rsc_op id="234" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="243" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -805,7 +805,7 @@
+ </synapse>
+ <synapse id="59">
+ <action_set>
+- <rsc_op id="229" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:2_pre_notify_stop_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="238" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:2_pre_notify_stop_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -843,10 +843,10 @@
+ <pseudo_event id="167" operation="notify" operation_key="redis-bundle-master_post_notify_demoted_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="237" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
++ <rsc_op id="246" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="239" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="248" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -876,10 +876,10 @@
+ <pseudo_event id="165" operation="notify" operation_key="redis-bundle-master_pre_notify_demote_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="236" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
++ <rsc_op id="245" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="238" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="247" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -932,13 +932,13 @@
+ <pseudo_event id="161" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="231" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1"/>
++ <rsc_op id="240" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="233" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
++ <rsc_op id="242" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="235" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="244" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -968,13 +968,13 @@
+ <pseudo_event id="159" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="230" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1"/>
++ <rsc_op id="239" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="232" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
++ <rsc_op id="241" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="234" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="243" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -1076,10 +1076,10 @@
+ <pseudo_event id="153" operation="notify" operation_key="redis-bundle-master_pre_notify_stop_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="228" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:1_pre_notify_stop_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
++ <rsc_op id="237" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:1_pre_notify_stop_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="229" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:2_pre_notify_stop_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="238" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:2_pre_notify_stop_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp
+index bf9a0b0..168ec7a 100644
+--- a/pengine/test10/bundle-order-partial-start-2.exp
++++ b/pengine/test10/bundle-order-partial-start-2.exp
+@@ -1,7 +1,7 @@
+ <transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0" priority="1000000">
+ <action_set>
+- <rsc_op id="130" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <rsc_op id="133" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
+ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="rabbitmq-bundle-0 undercloud" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="rabbitmq:0" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="rabbitmq:0" CRM_meta_notify_start_uname="rabbitmq-bundle-0" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
+ </rsc_op>
+@@ -61,7 +61,7 @@
+ <pseudo_event id="32" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_running_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="130" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ <rsc_op id="133" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -315,7 +315,7 @@
+ </synapse>
+ <synapse id="22" priority="1000000">
+ <action_set>
+- <rsc_op id="134" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <rsc_op id="137" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0" CRM_meta_notify_active_uname="redis-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="redis-bundle-0 undercloud" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0" CRM_meta_notify_slave_uname="redis-bundle-0" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -328,7 +328,7 @@
+ </synapse>
+ <synapse id="23">
+ <action_set>
+- <rsc_op id="133" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <rsc_op id="136" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0" CRM_meta_notify_active_uname="redis-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="redis-bundle-0 undercloud" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0" CRM_meta_notify_slave_uname="redis-bundle-0" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -379,7 +379,7 @@
+ <pseudo_event id="91" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="134" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ <rsc_op id="137" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -409,7 +409,7 @@
+ <pseudo_event id="89" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="133" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ <rsc_op id="136" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp
+index 8e28f19..5e80822 100644
+--- a/pengine/test10/bundle-order-partial-start.exp
++++ b/pengine/test10/bundle-order-partial-start.exp
+@@ -1,7 +1,7 @@
+ <transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0" priority="1000000">
+ <action_set>
+- <rsc_op id="129" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <rsc_op id="131" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
+ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="rabbitmq-bundle-0 undercloud" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="rabbitmq:0" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="rabbitmq:0" CRM_meta_notify_start_uname="rabbitmq-bundle-0" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
+ </rsc_op>
+@@ -61,7 +61,7 @@
+ <pseudo_event id="31" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_running_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="129" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ <rsc_op id="131" operation="notify" operation_key="rabbitmq:0_post_notify_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -296,7 +296,7 @@
+ </synapse>
+ <synapse id="21" priority="1000000">
+ <action_set>
+- <rsc_op id="133" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <rsc_op id="135" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0" CRM_meta_notify_active_uname="redis-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="redis-bundle-0 undercloud" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0" CRM_meta_notify_slave_uname="redis-bundle-0" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -309,7 +309,7 @@
+ </synapse>
+ <synapse id="22">
+ <action_set>
+- <rsc_op id="132" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <rsc_op id="134" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0" CRM_meta_notify_active_uname="redis-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="redis-bundle-0 undercloud" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0" CRM_meta_notify_slave_uname="redis-bundle-0" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -360,7 +360,7 @@
+ <pseudo_event id="90" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="133" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ <rsc_op id="135" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -390,7 +390,7 @@
+ <pseudo_event id="88" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="132" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ <rsc_op id="134" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+diff --git a/pengine/test10/bundle-order-partial-stop.exp b/pengine/test10/bundle-order-partial-stop.exp
+index 89d87aa..626fc44 100644
+--- a/pengine/test10/bundle-order-partial-stop.exp
++++ b/pengine/test10/bundle-order-partial-stop.exp
+@@ -1,7 +1,7 @@
+ <transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+- <rsc_op id="135" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <rsc_op id="138" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
+ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0" CRM_meta_notify_active_uname="rabbitmq-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="rabbitmq-bundle-0 undercloud" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
+ </rsc_op>
+@@ -66,7 +66,7 @@
+ <pseudo_event id="35" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="135" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ <rsc_op id="138" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -278,7 +278,7 @@
+ </synapse>
+ <synapse id="19" priority="1000000">
+ <action_set>
+- <rsc_op id="138" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:0_post_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <rsc_op id="141" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:0_post_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0" CRM_meta_notify_active_uname="redis-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="redis-bundle-0 undercloud" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -291,7 +291,7 @@
+ </synapse>
+ <synapse id="20">
+ <action_set>
+- <rsc_op id="137" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:0_pre_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <rsc_op id="140" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:0_pre_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0" CRM_meta_notify_active_uname="redis-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="redis-bundle-0 undercloud" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -304,7 +304,7 @@
+ </synapse>
+ <synapse id="21">
+ <action_set>
+- <rsc_op id="136" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:0_pre_notify_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <rsc_op id="139" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:0_pre_notify_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0" CRM_meta_notify_active_uname="redis-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="redis-bundle-0 undercloud" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -370,7 +370,7 @@
+ <pseudo_event id="95" operation="notify" operation_key="redis-bundle-master_post_notify_demoted_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="138" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:0_post_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ <rsc_op id="141" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:0_post_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -400,7 +400,7 @@
+ <pseudo_event id="93" operation="notify" operation_key="redis-bundle-master_pre_notify_demote_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="137" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:0_pre_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ <rsc_op id="140" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:0_pre_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -480,7 +480,7 @@
+ <pseudo_event id="81" operation="notify" operation_key="redis-bundle-master_pre_notify_stop_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="136" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:0_pre_notify_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ <rsc_op id="139" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:0_pre_notify_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+diff --git a/pengine/test10/bundle-order-stop-clone.exp b/pengine/test10/bundle-order-stop-clone.exp
+index 3e66f54..6ef5dac 100644
+--- a/pengine/test10/bundle-order-stop-clone.exp
++++ b/pengine/test10/bundle-order-stop-clone.exp
+@@ -1,7 +1,7 @@
+ <transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+- <rsc_op id="164" operation="notify" operation_key="storage:0_pre_notify_stop_0" on_node="metal-1" on_node_uuid="1">
++ <rsc_op id="170" operation="notify" operation_key="storage:0_pre_notify_stop_0" on_node="metal-1" on_node_uuid="1">
+ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="pre" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+@@ -27,7 +27,7 @@
+ </synapse>
+ <synapse id="2" priority="1000000">
+ <action_set>
+- <rsc_op id="166" operation="notify" operation_key="storage:1_post_notify_stop_0" on_node="metal-2" on_node_uuid="2">
++ <rsc_op id="172" operation="notify" operation_key="storage:1_post_notify_stop_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="post" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+@@ -40,7 +40,7 @@
+ </synapse>
+ <synapse id="3">
+ <action_set>
+- <rsc_op id="165" operation="notify" operation_key="storage:1_pre_notify_stop_0" on_node="metal-2" on_node_uuid="2">
++ <rsc_op id="171" operation="notify" operation_key="storage:1_pre_notify_stop_0" on_node="metal-2" on_node_uuid="2">
+ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="pre" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+@@ -53,7 +53,7 @@
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+- <rsc_op id="168" operation="notify" operation_key="storage:2_post_notify_stop_0" on_node="metal-3" on_node_uuid="3">
++ <rsc_op id="174" operation="notify" operation_key="storage:2_post_notify_stop_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="post" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+@@ -66,7 +66,7 @@
+ </synapse>
+ <synapse id="5">
+ <action_set>
+- <rsc_op id="167" operation="notify" operation_key="storage:2_pre_notify_stop_0" on_node="metal-3" on_node_uuid="3">
++ <rsc_op id="173" operation="notify" operation_key="storage:2_pre_notify_stop_0" on_node="metal-3" on_node_uuid="3">
+ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="pre" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
+ </rsc_op>
+@@ -88,10 +88,10 @@
+ <pseudo_event id="56" operation="notify" operation_key="storage-clone_post_notify_stopped_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="166" operation="notify" operation_key="storage:1_post_notify_stop_0" on_node="metal-2" on_node_uuid="2"/>
++ <rsc_op id="172" operation="notify" operation_key="storage:1_post_notify_stop_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="168" operation="notify" operation_key="storage:2_post_notify_stop_0" on_node="metal-3" on_node_uuid="3"/>
++ <rsc_op id="174" operation="notify" operation_key="storage:2_post_notify_stop_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -121,13 +121,13 @@
+ <pseudo_event id="54" operation="notify" operation_key="storage-clone_pre_notify_stop_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="164" operation="notify" operation_key="storage:0_pre_notify_stop_0" on_node="metal-1" on_node_uuid="1"/>
++ <rsc_op id="170" operation="notify" operation_key="storage:0_pre_notify_stop_0" on_node="metal-1" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="165" operation="notify" operation_key="storage:1_pre_notify_stop_0" on_node="metal-2" on_node_uuid="2"/>
++ <rsc_op id="171" operation="notify" operation_key="storage:1_pre_notify_stop_0" on_node="metal-2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="167" operation="notify" operation_key="storage:2_pre_notify_stop_0" on_node="metal-3" on_node_uuid="3"/>
++ <rsc_op id="173" operation="notify" operation_key="storage:2_pre_notify_stop_0" on_node="metal-3" on_node_uuid="3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+diff --git a/pengine/test10/bundle-order-stop-on-remote.exp b/pengine/test10/bundle-order-stop-on-remote.exp
+index 96588dc..6559dfd 100644
+--- a/pengine/test10/bundle-order-stop-on-remote.exp
++++ b/pengine/test10/bundle-order-stop-on-remote.exp
+@@ -588,7 +588,7 @@
+ </synapse>
+ <synapse id="34" priority="1000000">
+ <action_set>
+- <rsc_op id="245" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0">
++ <rsc_op id="254" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 database-0 database-1 database-2 database-0 database-1 database-2 messaging-0 messaging-1 messaging-2 messaging-0 messaging-1 messaging-2 controller-0 redis-bundle-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 database-0 database-1 database-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 messaging-0 messaging-1 messaging-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="database-1 database-0 controller-2 controller-1 controller-0 redis-bundle-2 redis-bundle-1 redis-bundle-0 messaging-2 messaging-1 messaging-0 database-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource="redis:1" CRM_meta_notify_start_uname="redis-bundle-1" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -601,7 +601,7 @@
+ </synapse>
+ <synapse id="35">
+ <action_set>
+- <rsc_op id="244" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0">
++ <rsc_op id="253" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 database-0 database-1 database-2 database-0 database-1 database-2 messaging-0 messaging-1 messaging-2 messaging-0 messaging-1 messaging-2 controller-0 redis-bundle-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 database-0 database-1 database-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 messaging-0 messaging-1 messaging-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="database-1 database-0 controller-2 controller-1 controller-0 redis-bundle-2 redis-bundle-1 redis-bundle-0 messaging-2 messaging-1 messaging-0 database-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource="redis:1" CRM_meta_notify_start_uname="redis-bundle-1" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -614,7 +614,7 @@
+ </synapse>
+ <synapse id="36" priority="1000000">
+ <action_set>
+- <rsc_op id="236" operation="notify" operation_key="redis_post_notify_start_0" internal_operation_key="redis:0_post_notify_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0">
++ <rsc_op id="245" operation="notify" operation_key="redis_post_notify_start_0" internal_operation_key="redis:0_post_notify_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 database-0 database-1 database-2 database-0 database-1 database-2 messaging-0 messaging-1 messaging-2 messaging-0 messaging-1 messaging-2 controller-0 redis-bundle-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 database-0 database-1 database-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 messaging-0 messaging-1 messaging-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="database-1 database-0 controller-2 controller-1 controller-0 redis-bundle-2 redis-bundle-1 redis-bundle-0 messaging-2 messaging-1 messaging-0 database-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource="redis:1" CRM_meta_notify_start_uname="redis-bundle-1" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -627,7 +627,7 @@
+ </synapse>
+ <synapse id="37">
+ <action_set>
+- <rsc_op id="235" operation="notify" operation_key="redis_pre_notify_start_0" internal_operation_key="redis:0_pre_notify_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0">
++ <rsc_op id="244" operation="notify" operation_key="redis_pre_notify_start_0" internal_operation_key="redis:0_pre_notify_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 database-0 database-1 database-2 database-0 database-1 database-2 messaging-0 messaging-1 messaging-2 messaging-0 messaging-1 messaging-2 controller-0 redis-bundle-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 database-0 database-1 database-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 messaging-0 messaging-1 messaging-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="database-1 database-0 controller-2 controller-1 controller-0 redis-bundle-2 redis-bundle-1 redis-bundle-0 messaging-2 messaging-1 messaging-0 database-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource="redis:1" CRM_meta_notify_start_uname="redis-bundle-1" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -675,7 +675,7 @@
+ </synapse>
+ <synapse id="40" priority="1000000">
+ <action_set>
+- <rsc_op id="247" operation="notify" operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="256" operation="notify" operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 database-0 database-1 database-2 database-0 database-1 database-2 messaging-0 messaging-1 messaging-2 messaging-0 messaging-1 messaging-2 controller-0 redis-bundle-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 database-0 database-1 database-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 messaging-0 messaging-1 messaging-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="database-1 database-0 controller-2 controller-1 controller-0 redis-bundle-2 redis-bundle-1 redis-bundle-0 messaging-2 messaging-1 messaging-0 database-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource="redis:1" CRM_meta_notify_start_uname="redis-bundle-1" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -688,7 +688,7 @@
+ </synapse>
+ <synapse id="41">
+ <action_set>
+- <rsc_op id="246" operation="notify" operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="255" operation="notify" operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 database-0 database-1 database-2 database-0 database-1 database-2 messaging-0 messaging-1 messaging-2 messaging-0 messaging-1 messaging-2 controller-0 redis-bundle-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 database-0 database-1 database-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 messaging-0 messaging-1 messaging-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="database-1 database-0 controller-2 controller-1 controller-0 redis-bundle-2 redis-bundle-1 redis-bundle-0 messaging-2 messaging-1 messaging-0 database-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource="redis:1" CRM_meta_notify_start_uname="redis-bundle-1" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -701,7 +701,7 @@
+ </synapse>
+ <synapse id="42" priority="1000000">
+ <action_set>
+- <rsc_op id="237" operation="notify" operation_key="redis:1_post_notify_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="246" operation="notify" operation_key="redis:1_post_notify_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 database-0 database-1 database-2 database-0 database-1 database-2 messaging-0 messaging-1 messaging-2 messaging-0 messaging-1 messaging-2 controller-0 redis-bundle-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 database-0 database-1 database-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 messaging-0 messaging-1 messaging-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="database-1 database-0 controller-2 controller-1 controller-0 redis-bundle-2 redis-bundle-1 redis-bundle-0 messaging-2 messaging-1 messaging-0 database-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource="redis:1" CRM_meta_notify_start_uname="redis-bundle-1" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -780,7 +780,7 @@
+ </synapse>
+ <synapse id="46" priority="1000000">
+ <action_set>
+- <rsc_op id="249" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="258" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 database-0 database-1 database-2 database-0 database-1 database-2 messaging-0 messaging-1 messaging-2 messaging-0 messaging-1 messaging-2 controller-0 redis-bundle-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 database-0 database-1 database-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 messaging-0 messaging-1 messaging-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="database-1 database-0 controller-2 controller-1 controller-0 redis-bundle-2 redis-bundle-1 redis-bundle-0 messaging-2 messaging-1 messaging-0 database-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource="redis:1" CRM_meta_notify_start_uname="redis-bundle-1" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -793,7 +793,7 @@
+ </synapse>
+ <synapse id="47">
+ <action_set>
+- <rsc_op id="248" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="257" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 database-0 database-1 database-2 database-0 database-1 database-2 messaging-0 messaging-1 messaging-2 messaging-0 messaging-1 messaging-2 controller-0 redis-bundle-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 database-0 database-1 database-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 messaging-0 messaging-1 messaging-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="database-1 database-0 controller-2 controller-1 controller-0 redis-bundle-2 redis-bundle-1 redis-bundle-0 messaging-2 messaging-1 messaging-0 database-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource="redis:1" CRM_meta_notify_start_uname="redis-bundle-1" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -806,7 +806,7 @@
+ </synapse>
+ <synapse id="48" priority="1000000">
+ <action_set>
+- <rsc_op id="239" operation="notify" operation_key="redis_post_notify_start_0" internal_operation_key="redis:2_post_notify_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="248" operation="notify" operation_key="redis_post_notify_start_0" internal_operation_key="redis:2_post_notify_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 database-0 database-1 database-2 database-0 database-1 database-2 messaging-0 messaging-1 messaging-2 messaging-0 messaging-1 messaging-2 controller-0 redis-bundle-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 database-0 database-1 database-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 messaging-0 messaging-1 messaging-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="database-1 database-0 controller-2 controller-1 controller-0 redis-bundle-2 redis-bundle-1 redis-bundle-0 messaging-2 messaging-1 messaging-0 database-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource="redis:1" CRM_meta_notify_start_uname="redis-bundle-1" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -819,7 +819,7 @@
+ </synapse>
+ <synapse id="49">
+ <action_set>
+- <rsc_op id="238" operation="notify" operation_key="redis_pre_notify_start_0" internal_operation_key="redis:2_pre_notify_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="247" operation="notify" operation_key="redis_pre_notify_start_0" internal_operation_key="redis:2_pre_notify_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 database-0 database-1 database-2 database-0 database-1 database-2 messaging-0 messaging-1 messaging-2 messaging-0 messaging-1 messaging-2 controller-0 redis-bundle-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 database-0 database-1 database-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 messaging-0 messaging-1 messaging-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="database-1 database-0 controller-2 controller-1 controller-0 redis-bundle-2 redis-bundle-1 redis-bundle-0 messaging-2 messaging-1 messaging-0 database-2" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource="redis:1" CRM_meta_notify_start_uname="redis-bundle-1" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -841,13 +841,13 @@
+ <pseudo_event id="169" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="245" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0"/>
++ <rsc_op id="254" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="247" operation="notify" operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
++ <rsc_op id="256" operation="notify" operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="249" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="258" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -877,13 +877,13 @@
+ <pseudo_event id="167" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="244" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0"/>
++ <rsc_op id="253" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="246" operation="notify" operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
++ <rsc_op id="255" operation="notify" operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="248" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="257" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -940,13 +940,13 @@
+ <pseudo_event id="157" operation="notify" operation_key="redis-bundle-master_post_notify_running_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="236" operation="notify" operation_key="redis_post_notify_start_0" internal_operation_key="redis:0_post_notify_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0"/>
++ <rsc_op id="245" operation="notify" operation_key="redis_post_notify_start_0" internal_operation_key="redis:0_post_notify_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="237" operation="notify" operation_key="redis:1_post_notify_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
++ <rsc_op id="246" operation="notify" operation_key="redis:1_post_notify_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="239" operation="notify" operation_key="redis_post_notify_start_0" internal_operation_key="redis:2_post_notify_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="248" operation="notify" operation_key="redis_post_notify_start_0" internal_operation_key="redis:2_post_notify_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -976,10 +976,10 @@
+ <pseudo_event id="155" operation="notify" operation_key="redis-bundle-master_pre_notify_start_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="235" operation="notify" operation_key="redis_pre_notify_start_0" internal_operation_key="redis:0_pre_notify_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0"/>
++ <rsc_op id="244" operation="notify" operation_key="redis_pre_notify_start_0" internal_operation_key="redis:0_pre_notify_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="238" operation="notify" operation_key="redis_pre_notify_start_0" internal_operation_key="redis:2_pre_notify_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="247" operation="notify" operation_key="redis_pre_notify_start_0" internal_operation_key="redis:2_pre_notify_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+diff --git a/pengine/test10/bundle-order-stop.exp b/pengine/test10/bundle-order-stop.exp
+index 89d87aa..626fc44 100644
+--- a/pengine/test10/bundle-order-stop.exp
++++ b/pengine/test10/bundle-order-stop.exp
+@@ -1,7 +1,7 @@
+ <transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+- <rsc_op id="135" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <rsc_op id="138" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
+ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0" CRM_meta_notify_active_uname="rabbitmq-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="rabbitmq-bundle-0 undercloud" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
+ </rsc_op>
+@@ -66,7 +66,7 @@
+ <pseudo_event id="35" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="135" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ <rsc_op id="138" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -278,7 +278,7 @@
+ </synapse>
+ <synapse id="19" priority="1000000">
+ <action_set>
+- <rsc_op id="138" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:0_post_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <rsc_op id="141" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:0_post_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0" CRM_meta_notify_active_uname="redis-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="redis-bundle-0 undercloud" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -291,7 +291,7 @@
+ </synapse>
+ <synapse id="20">
+ <action_set>
+- <rsc_op id="137" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:0_pre_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <rsc_op id="140" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:0_pre_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0" CRM_meta_notify_active_uname="redis-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="redis-bundle-0 undercloud" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -304,7 +304,7 @@
+ </synapse>
+ <synapse id="21">
+ <action_set>
+- <rsc_op id="136" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:0_pre_notify_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <rsc_op id="139" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:0_pre_notify_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0" CRM_meta_notify_active_uname="redis-bundle-0" CRM_meta_notify_all_uname="galera-bundle-0 rabbitmq-bundle-0 redis-bundle-0 undercloud" CRM_meta_notify_available_uname="redis-bundle-0 undercloud" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="undercloud" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -370,7 +370,7 @@
+ <pseudo_event id="95" operation="notify" operation_key="redis-bundle-master_post_notify_demoted_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="138" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:0_post_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ <rsc_op id="141" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:0_post_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -400,7 +400,7 @@
+ <pseudo_event id="93" operation="notify" operation_key="redis-bundle-master_pre_notify_demote_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="137" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:0_pre_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ <rsc_op id="140" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:0_pre_notify_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -480,7 +480,7 @@
+ <pseudo_event id="81" operation="notify" operation_key="redis-bundle-master_pre_notify_stop_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="136" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:0_pre_notify_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ <rsc_op id="139" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:0_pre_notify_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+ </synapse>
+--
+1.8.3.1
+
+
+From 16fda11606c3cbc432153a8677ab7c378f0cbd2e Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Tue, 9 Oct 2018 11:57:43 -0500
+Subject: [PATCH 3/9] Log: scheduler: improve bundle address fixing messages
+
+Mostly, try to make clear when the bundle container's node is used vs.
+the bundle connection's node.
+---
+ lib/pengine/container.c | 5 +++--
+ lib/pengine/utils.c | 3 ++-
+ pengine/container.c | 6 ++++--
+ 3 files changed, 9 insertions(+), 5 deletions(-)
+
+diff --git a/lib/pengine/container.c b/lib/pengine/container.c
+index 1526f37..483d219 100644
+--- a/lib/pengine/container.c
++++ b/lib/pengine/container.c
+@@ -814,11 +814,12 @@ container_fix_remote_addr_in(resource_t *rsc, xmlNode *xml, const char *field)
+ }
+
+ if(node == NULL) {
+- crm_trace("Cannot fix address for %s", tuple->remote->id);
++ crm_trace("Cannot determine address for bundle connection %s", rsc->id);
+ return NULL;
+ }
+
+- crm_trace("Fixing addr for %s on %s", rsc->id, node->details->uname);
++ crm_trace("Setting address for bundle connection %s to bundle host %s",
++ rsc->id, node->details->uname);
+ if(xml != NULL && field != NULL) {
+ crm_xml_add(xml, field, node->details->uname);
+ }
+diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
+index a9ca86f..3f8dc30 100644
+--- a/lib/pengine/utils.c
++++ b/lib/pengine/utils.c
+@@ -1971,7 +1971,8 @@ rsc_action_digest(resource_t * rsc, const char *task, const char *key,
+
+ // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
+ if (container_fix_remote_addr_in(rsc, data->params_all, "addr")) {
+- crm_trace("Fixed addr for %s on %s", rsc->id, node->details->uname);
++ crm_trace("Set address for bundle connection %s (on %s)",
++ rsc->id, node->details->uname);
+ }
+
+ g_hash_table_foreach(local_rsc_params, hash2field, data->params_all);
+diff --git a/pengine/container.c b/pengine/container.c
+index 4e77545..02ee425 100644
+--- a/pengine/container.c
++++ b/pengine/container.c
+@@ -846,10 +846,12 @@ container_expand(resource_t * rsc, pe_working_set_t * data_set)
+ const char *calculated_addr = container_fix_remote_addr_in(tuple->remote, nvpair, "value");
+
+ if (calculated_addr) {
+- crm_trace("Fixed addr for %s on %s", tuple->remote->id, calculated_addr);
++ crm_trace("Set address for bundle connection %s to bundle host %s",
++ tuple->remote->id, calculated_addr);
+ g_hash_table_replace(tuple->remote->parameters, strdup("addr"), strdup(calculated_addr));
+ } else {
+- crm_err("Could not fix addr for %s", tuple->remote->id);
++ crm_err("Could not determine address for bundle connection %s",
++ tuple->remote->id);
+ }
+ }
+ if(tuple->ip) {
+--
+1.8.3.1
+
+
+From 0907d0e3061f05d0fdd3a276e8a5c578813d5ea9 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Tue, 9 Oct 2018 16:09:33 -0500
+Subject: [PATCH 4/9] Refactor: scheduler: remove redundant code
+
+unpack_rsc_op() doesn't need to map status PCMK_LRM_OP_ERROR to
+PCMK_LRM_OP_DONE because it explicitly looks for both when it uses status
+
+check_operation_expiry() doesn't need to check that failure_timeout is positive
+when expired is true, because expired can be true only if failure_timeout is
+positive
+
+check_action_definition() doesn't need to check whether task is stop because
+it cannot be called for stops
+
+check_actions_for() doesn't need to check whether a status operation is a probe
+because it also checks for recurring operations in the same place
+---
+ lib/pengine/unpack.c | 32 +++++++++++++++-----------------
+ pengine/allocate.c | 14 ++++----------
+ 2 files changed, 19 insertions(+), 27 deletions(-)
+
+diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
+index 52518d4..7bc78a3 100644
+--- a/lib/pengine/unpack.c
++++ b/lib/pengine/unpack.c
+@@ -2990,21 +2990,24 @@ static bool check_operation_expiry(resource_t *rsc, node_t *node, int rc, xmlNod
+ }
+
+ if (expired) {
+- if (failure_timeout > 0) {
+- if (pe_get_failcount(node, rsc, &last_failure, pe_fc_default,
+- xml_op, data_set)) {
++ if (pe_get_failcount(node, rsc, &last_failure, pe_fc_default, xml_op,
++ data_set)) {
+
+- if (pe_get_failcount(node, rsc, &last_failure, pe_fc_effective,
+- xml_op, data_set) == 0) {
+- clear_reason = "it expired";
+- } else {
+- expired = FALSE;
+- }
++ // There is a fail count ignoring timeout
+
+- } else if (rsc->remote_reconnect_interval && strstr(ID(xml_op), "last_failure")) {
+- /* always clear last failure when reconnect interval is set */
+- clear_reason = "reconnect interval is set";
++ if (pe_get_failcount(node, rsc, &last_failure, pe_fc_effective,
++ xml_op, data_set) == 0) {
++ // There is no fail count considering timeout
++ clear_reason = "it expired";
++
++ } else {
++ expired = FALSE;
+ }
++
++ } else if (rsc->remote_reconnect_interval
++ && strstr(ID(xml_op), "last_failure")) {
++ // Always clear last failure when reconnect interval is set
++ clear_reason = "reconnect interval is set";
+ }
+
+ } else if (strstr(ID(xml_op), "last_failure") &&
+@@ -3240,11 +3243,6 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last
+ node->details->uname, rsc->id);
+ }
+
+- if (status == PCMK_LRM_OP_ERROR) {
+- /* Older versions set this if rc != 0 but it's up to us to decide */
+- status = PCMK_LRM_OP_DONE;
+- }
+-
+ if(status != PCMK_LRM_OP_NOT_INSTALLED) {
+ expired = check_operation_expiry(rsc, node, rc, xml_op, data_set);
+ }
+diff --git a/pengine/allocate.c b/pengine/allocate.c
+index dc8017a..5589a2f 100644
+--- a/pengine/allocate.c
++++ b/pengine/allocate.c
+@@ -262,9 +262,6 @@ check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op
+ const char *digest_secure = NULL;
+
+ CRM_CHECK(active_node != NULL, return FALSE);
+- if (safe_str_eq(task, RSC_STOP)) {
+- return FALSE;
+- }
+
+ interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
+ interval = crm_parse_int(interval_s, "0");
+@@ -395,7 +392,6 @@ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_worki
+ xmlNode *rsc_op = NULL;
+ GListPtr op_list = NULL;
+ GListPtr sorted_op_list = NULL;
+- gboolean is_probe = FALSE;
+ gboolean did_change = FALSE;
+
+ CRM_CHECK(node != NULL, return);
+@@ -449,22 +445,20 @@ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_worki
+ continue;
+ }
+
+- is_probe = FALSE;
+ did_change = FALSE;
+ task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
+
+ interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
+ interval = crm_parse_int(interval_s, "0");
+
+- if (interval == 0 && safe_str_eq(task, RSC_STATUS)) {
+- is_probe = TRUE;
+- }
+-
+ if (interval > 0 &&
+ (is_set(rsc->flags, pe_rsc_maintenance) || node->details->maintenance)) {
+ CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);
+
+- } else if (is_probe || safe_str_eq(task, RSC_START) || safe_str_eq(task, RSC_PROMOTE) || interval > 0
++ } else if ((interval > 0)
++ || safe_str_eq(task, RSC_STATUS)
++ || safe_str_eq(task, RSC_START)
++ || safe_str_eq(task, RSC_PROMOTE)
+ || safe_str_eq(task, RSC_MIGRATED)) {
+ did_change = check_action_definition(rsc, node, rsc_op, data_set);
+ }
+--
+1.8.3.1
+
+
+From 83b6e8d1453dc6656384bbde3bfb86c5970d54c2 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Tue, 9 Oct 2018 15:04:24 -0500
+Subject: [PATCH 5/9] Refactor: scheduler: functionize scheduling clearing of
+ fail count
+
+Reduces duplication, allows reuse, and improves consistency (for example,
+some uses previously set XML_ATTR_TE_NOWAIT and some didn't).
+---
+ include/crm/pengine/internal.h | 3 +++
+ lib/pengine/failcounts.c | 30 +++++++++++++++++++++++++++-
+ lib/pengine/unpack.c | 10 ++--------
+ pengine/allocate.c | 44 +++++++++++++-----------------------------
+ 4 files changed, 47 insertions(+), 40 deletions(-)
+
+diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
+index 4aca751..6745ae3 100644
+--- a/include/crm/pengine/internal.h
++++ b/include/crm/pengine/internal.h
+@@ -114,6 +114,9 @@ int pe_get_failcount(node_t *node, resource_t *rsc, time_t *last_failure,
+ uint32_t flags, xmlNode *xml_op,
+ pe_working_set_t *data_set);
+
++pe_action_t *pe__clear_failcount(pe_resource_t *rsc, pe_node_t *node,
++ const char *reason,
++ pe_working_set_t *data_set);
+
+ /* Functions for finding/counting a resource's active nodes */
+
+diff --git a/lib/pengine/failcounts.c b/lib/pengine/failcounts.c
+index e217176..8a7d0e4 100644
+--- a/lib/pengine/failcounts.c
++++ b/lib/pengine/failcounts.c
+@@ -1,5 +1,5 @@
+ /*
+- * Copyright (C) 2008-2017 Andrew Beekhof <andrew@beekhof.net>
++ * Copyright 2008-2018 Andrew Beekhof <andrew@beekhof.net>
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+@@ -319,3 +319,31 @@ pe_get_failcount(node_t *node, resource_t *rsc, time_t *last_failure,
+
+ return failcount;
+ }
++
++/*!
++ * \brief Schedule a controller operation to clear a fail count
++ *
++ * \param[in] rsc Resource with failure
++ * \param[in] node Node failure occurred on
++ * \param[in] reason Readable description why needed (for logging)
++ * \param[in] data_set Working set for cluster
++ *
++ * \return Scheduled action
++ */
++pe_action_t *
++pe__clear_failcount(pe_resource_t *rsc, pe_node_t *node,
++ const char *reason, pe_working_set_t *data_set)
++{
++ char *key = NULL;
++ action_t *clear = NULL;
++
++ CRM_CHECK(rsc && node && reason && data_set, return NULL);
++
++ key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
++ clear = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE,
++ data_set);
++ add_hash_param(clear->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
++ crm_notice("Clearing failure of %s on %s because %s " CRM_XS " %s",
++ rsc->id, node->details->uname, reason, clear->uuid);
++ return clear;
++}
+diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
+index 7bc78a3..8f9099c 100644
+--- a/lib/pengine/unpack.c
++++ b/lib/pengine/unpack.c
+@@ -3031,14 +3031,8 @@ static bool check_operation_expiry(resource_t *rsc, node_t *node, int rc, xmlNod
+
+ if (clear_reason != NULL) {
+ node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);
+- char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
+- action_t *clear_op = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT,
+- node, FALSE, TRUE, data_set);
+-
+- add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
+-
+- crm_notice("Clearing failure of %s on %s because %s " CRM_XS " %s",
+- rsc->id, node->details->uname, clear_reason, clear_op->uuid);
++ pe_action_t *clear_op = pe__clear_failcount(rsc, node, clear_reason,
++ data_set);
+
+ if (is_set(data_set->flags, pe_flag_stonith_enabled)
+ && rsc->remote_reconnect_interval
+diff --git a/pengine/allocate.c b/pengine/allocate.c
+index 5589a2f..569a4a5 100644
+--- a/pengine/allocate.c
++++ b/pengine/allocate.c
+@@ -376,7 +376,6 @@ check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op
+ return did_change;
+ }
+
+-
+ static void
+ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_working_set_t * data_set)
+ {
+@@ -392,7 +391,6 @@ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_worki
+ xmlNode *rsc_op = NULL;
+ GListPtr op_list = NULL;
+ GListPtr sorted_op_list = NULL;
+- gboolean did_change = FALSE;
+
+ CRM_CHECK(node != NULL, return);
+
+@@ -445,7 +443,6 @@ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_worki
+ continue;
+ }
+
+- did_change = FALSE;
+ task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
+
+ interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
+@@ -453,6 +450,7 @@ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_worki
+
+ if (interval > 0 &&
+ (is_set(rsc->flags, pe_rsc_maintenance) || node->details->maintenance)) {
++ // Maintenance mode cancels recurring operations
+ CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);
+
+ } else if ((interval > 0)
+@@ -460,28 +458,18 @@ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_worki
+ || safe_str_eq(task, RSC_START)
+ || safe_str_eq(task, RSC_PROMOTE)
+ || safe_str_eq(task, RSC_MIGRATED)) {
+- did_change = check_action_definition(rsc, node, rsc_op, data_set);
+- }
+-
+- if (did_change && pe_get_failcount(node, rsc, NULL, pe_fc_effective,
+- NULL, data_set)) {
+-
+- char *key = NULL;
+- action_t *action_clear = NULL;
+-
+- key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
+- action_clear =
+- custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);
+- set_bit(action_clear->flags, pe_action_runnable);
+-
+- crm_notice("Clearing failure of %s on %s "
+- "because action definition changed " CRM_XS " %s",
+- rsc->id, node->details->uname, action_clear->uuid);
++ /* If a resource operation failed, and the operation's definition
++ * has changed, clear any fail count so they can be retried fresh.
++ */
++ if (check_action_definition(rsc, node, rsc_op, data_set)
++ && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
++ data_set)) {
++ pe__clear_failcount(rsc, node, "action definition changed",
++ data_set);
++ }
+ }
+ }
+-
+ g_list_free(sorted_op_list);
+-
+ }
+
+ static GListPtr
+@@ -1254,16 +1242,10 @@ cleanup_orphans(resource_t * rsc, pe_working_set_t * data_set)
+ && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
+ data_set)) {
+
+- char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
+- action_t *clear_op = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT,
+- node, FALSE, TRUE, data_set);
+-
+- add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
++ pe_action_t *clear_op = NULL;
+
+- pe_rsc_info(rsc,
+- "Clearing failure of %s on %s because it is orphaned "
+- CRM_XS " %s",
+- rsc->id, node->details->uname, clear_op->uuid);
++ clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
++ data_set);
+
+ /* We can't use order_action_then_stop() here because its
+ * pe_order_preserve breaks things
+--
+1.8.3.1
+
+
+From fade22882d52ac743b99adc4cbd98780b21e250b Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Mon, 15 Oct 2018 10:26:46 -0500
+Subject: [PATCH 6/9] Fix: scheduler: avoid unnecessary recovery of cleaned
+ guest nodes
+
+Generalize Andrew Beekhof's fix for bundle nodes in 407f524 to all guest nodes.
+Fixes RHBZ#1448467
+---
+ pengine/container.c | 1 -
+ pengine/native.c | 39 +++++++++++++++++++++++++--------------
+ 2 files changed, 25 insertions(+), 15 deletions(-)
+
+diff --git a/pengine/container.c b/pengine/container.c
+index 02ee425..a35763b 100644
+--- a/pengine/container.c
++++ b/pengine/container.c
+@@ -278,7 +278,6 @@ container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
+ order_start_start(rsc, tuple->docker, pe_order_runnable_left | pe_order_implies_first_printed);
+
+ if(tuple->child) {
+- new_rsc_order(tuple->docker, RSC_STATUS, tuple->remote, RSC_STOP, pe_order_optional, data_set);
+ order_stop_stop(rsc, tuple->child, pe_order_implies_first_printed);
+ }
+ order_stop_stop(rsc, tuple->docker, pe_order_implies_first_printed);
+diff --git a/pengine/native.c b/pengine/native.c
+index 6447234..cd746b6 100644
+--- a/pengine/native.c
++++ b/pengine/native.c
+@@ -1486,6 +1486,26 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
+ if (rsc->container) {
+ resource_t *remote_rsc = NULL;
+
++ if (rsc->is_remote_node) {
++ // rsc is the implicit remote connection for a guest or bundle node
++
++ /* Do not allow a guest resource to live on a Pacemaker Remote node,
++ * to avoid nesting remotes. However, allow bundles to run on remote
++ * nodes.
++ */
++ if (is_not_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
++ rsc_avoids_remote_nodes(rsc->container);
++ }
++
++ /* If someone cleans up a guest or bundle node's container, we will
++ * likely schedule a (re-)probe of the container and recovery of the
++ * connection. Order the connection stop after the container probe,
++ * so that if we detect the container running, we will trigger a new
++ * transition and avoid the unnecessary recovery.
++ */
++ new_rsc_order(rsc->container, RSC_STATUS, rsc, RSC_STOP,
++ pe_order_optional, data_set);
++
+ /* A user can specify that a resource must start on a Pacemaker Remote
+ * node by explicitly configuring it with the container=NODENAME
+ * meta-attribute. This is of questionable merit, since location
+@@ -1493,16 +1513,15 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
+ * we check whether a resource (that is not itself a remote connection)
+ * has container set to a remote node or guest node resource.
+ */
+- if (rsc->container->is_remote_node) {
++ } else if (rsc->container->is_remote_node) {
+ remote_rsc = rsc->container;
+- } else if (rsc->is_remote_node == FALSE) {
++ } else {
+ remote_rsc = rsc_contains_remote_node(data_set, rsc->container);
+ }
+
+ if (remote_rsc) {
+- /* The container represents a Pacemaker Remote node, so force the
+- * resource on the Pacemaker Remote node instead of colocating the
+- * resource with the container resource.
++ /* Force the resource on the Pacemaker Remote node instead of
++ * colocating the resource with the container resource.
+ */
+ GHashTableIter iter;
+ node_t *node = NULL;
+@@ -1512,6 +1531,7 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
+ node->weight = -INFINITY;
+ }
+ }
++
+ } else {
+ /* This resource is either a filler for a container that does NOT
+ * represent a Pacemaker Remote node, or a Pacemaker Remote
+@@ -1545,15 +1565,6 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
+ * or remote connection resources.*/
+ rsc_avoids_remote_nodes(rsc);
+ }
+-
+- /* If this is a guest node's implicit remote connection, do not allow the
+- * guest resource to live on a Pacemaker Remote node, to avoid nesting
+- * remotes. However, allow bundles to run on remote nodes.
+- */
+- if (rsc->is_remote_node && rsc->container
+- && is_not_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
+- rsc_avoids_remote_nodes(rsc->container);
+- }
+ }
+
+ void
+--
+1.8.3.1
+
+
+From af4f6a1cc13b92bba8109007a1fac809e0d80887 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Mon, 15 Oct 2018 12:21:14 -0500
+Subject: [PATCH 7/9] Fix: scheduler: order guest pseudo-fencing properly after
+ clean-up
+
+If the resource history of a guest node's container has been cleaned, we will
+schedule a (pseudo-)fence of the guest node, and a stop of the guest node's
+connection resource, but not a stop of the container (which appears already
+stopped). In this case, order the pseudo-fence after the connection stop, so we
+don't call remote_node_down() unless the container is really down (the
+connection stop will be avoided if the container is really up).
+---
+ pengine/allocate.c | 26 ++++++++++++++++++++++----
+ 1 file changed, 22 insertions(+), 4 deletions(-)
+
+diff --git a/pengine/allocate.c b/pengine/allocate.c
+index 569a4a5..e867368 100644
+--- a/pengine/allocate.c
++++ b/pengine/allocate.c
+@@ -1446,12 +1446,30 @@ fence_guest(pe_node_t *node, pe_action_t *done, pe_working_set_t *data_set)
+ node->details->uname, stonith_op->id,
+ container->id, stop->id);
+ } else {
+- crm_info("Implying guest node %s is down (action %d) ",
+- node->details->uname, stonith_op->id);
++ /* If we're fencing the guest node but there's no stop for the guest
++ * resource, we must think the guest is already stopped. However, we may
++ * think so because its resource history was just cleaned. To avoid
++ * unnecessarily considering the guest node down if it's really up,
++ * order the pseudo-fencing after any stop of the connection resource,
++ * which will be ordered after any container (re-)probe.
++ */
++ stop = find_first_action(node->details->remote_rsc->actions, NULL,
++ RSC_STOP, NULL);
++
++ if (stop) {
++ order_actions(stop, stonith_op, pe_order_optional);
++ crm_info("Implying guest node %s is down (action %d) "
++ "after connection is stopped (action %d)",
++ node->details->uname, stonith_op->id, stop->id);
++ } else {
++ /* Not sure why we're fencing, but everything must already be
++ * cleanly stopped.
++ */
++ crm_info("Implying guest node %s is down (action %d) ",
++ node->details->uname, stonith_op->id);
++ }
+ }
+
+- /* @TODO: Order pseudo-fence after any (optional) fence of guest's host */
+-
+ /* Order/imply other actions relative to pseudo-fence as with real fence */
+ stonith_constraints(node, stonith_op, data_set);
+ if(done) {
+--
+1.8.3.1
+
+
+From 4c771013389f395e16165312993597064d06d149 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Mon, 29 Oct 2018 14:56:47 -0500
+Subject: [PATCH 8/9] Test: pengine: update regression tests for recent changes
+
+clear-failcount change: operations in generated graphs now consistently have
+XML_ATTR_TE_NOWAIT
+
+guest node change: insignificant change in op numbering due to newly added
+optional constraint
+---
+ pengine/test10/bug-5025-1.exp | 2 +-
+ pengine/test10/bug-5025-3.exp | 2 +-
+ pengine/test10/bug-5069-op-disabled.exp | 2 +-
+ pengine/test10/bug-cl-5247.exp | 12 ++++++------
+ 4 files changed, 9 insertions(+), 9 deletions(-)
+
+diff --git a/pengine/test10/bug-5025-1.exp b/pengine/test10/bug-5025-1.exp
+index 053ece4..2a82e79 100644
+--- a/pengine/test10/bug-5025-1.exp
++++ b/pengine/test10/bug-5025-1.exp
+@@ -3,7 +3,7 @@
+ <action_set>
+ <crm_event id="4" operation="clear_failcount" operation_key="A_clear_failcount_0" on_node="fc16-builder" on_node_uuid="fc16-builder">
+ <primitive id="A" class="ocf" provider="pacemaker" type="Dummy"/>
+- <attributes CRM_meta_on_node="fc16-builder" CRM_meta_on_node_uuid="fc16-builder" CRM_meta_timeout="20000" allow-migrate="1" />
++ <attributes CRM_meta_on_node="fc16-builder" CRM_meta_on_node_uuid="fc16-builder" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000" allow-migrate="1" />
+ </crm_event>
+ </action_set>
+ <inputs/>
+diff --git a/pengine/test10/bug-5025-3.exp b/pengine/test10/bug-5025-3.exp
+index eb2e2e6..9360ca7 100644
+--- a/pengine/test10/bug-5025-3.exp
++++ b/pengine/test10/bug-5025-3.exp
+@@ -12,7 +12,7 @@
+ <action_set>
+ <crm_event id="4" operation="clear_failcount" operation_key="A_clear_failcount_0" on_node="fc16-builder" on_node_uuid="fc16-builder">
+ <primitive id="A" class="ocf" provider="pacemaker" type="Dummy"/>
+- <attributes CRM_meta_on_node="fc16-builder" CRM_meta_on_node_uuid="fc16-builder" CRM_meta_timeout="20000" allow-migrate="1" state="anywhere" target-role="started"/>
++ <attributes CRM_meta_on_node="fc16-builder" CRM_meta_on_node_uuid="fc16-builder" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000" allow-migrate="1" state="anywhere" target-role="started"/>
+ </crm_event>
+ </action_set>
+ <inputs/>
+diff --git a/pengine/test10/bug-5069-op-disabled.exp b/pengine/test10/bug-5069-op-disabled.exp
+index 9653af1..fffb8c4 100644
+--- a/pengine/test10/bug-5069-op-disabled.exp
++++ b/pengine/test10/bug-5069-op-disabled.exp
+@@ -3,7 +3,7 @@
+ <action_set>
+ <crm_event id="3" operation="clear_failcount" operation_key="A_clear_failcount_0" on_node="fc16-builder2" on_node_uuid="fc16-builder2">
+ <primitive id="A" class="ocf" provider="pacemaker" type="Dummy"/>
+- <attributes CRM_meta_on_node="fc16-builder2" CRM_meta_on_node_uuid="fc16-builder2" CRM_meta_timeout="20000" />
++ <attributes CRM_meta_on_node="fc16-builder2" CRM_meta_on_node_uuid="fc16-builder2" CRM_meta_op_no_wait="true" CRM_meta_timeout="20000" />
+ </crm_event>
+ </action_set>
+ <inputs/>
+diff --git a/pengine/test10/bug-cl-5247.exp b/pengine/test10/bug-cl-5247.exp
+index c21ed7d..5315360 100644
+--- a/pengine/test10/bug-cl-5247.exp
++++ b/pengine/test10/bug-cl-5247.exp
+@@ -451,7 +451,7 @@
+ </synapse>
+ <synapse id="31" priority="1000000">
+ <action_set>
+- <rsc_op id="81" operation="notify" operation_key="pgsql_post_notify_demote_0" internal_operation_key="pgsql:1_post_notify_demote_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3">
++ <rsc_op id="83" operation="notify" operation_key="pgsql_post_notify_demote_0" internal_operation_key="pgsql:1_post_notify_demote_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3">
+ <primitive id="pgsql" long-id="pgsql:1" class="ocf" provider="heartbeat" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="pgsql:0 pgsql:1" CRM_meta_notify_active_uname="pgsr02 pgsr01" CRM_meta_notify_all_uname="bl460g8n3 bl460g8n4 pgsr01 pgsr02" CRM_meta_notify_available_uname="bl460g8n4 bl460g8n3 pgsr01 pgsr02" CRM_meta_notify_demote_resource="pgsql:0" CRM_meta_notify_demote_uname="pgsr02" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="pgsql:0 pgsql:1" CRM_meta_notify_master_uname="pgsr02 pgsr01" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="pgsql:0" CRM_meta_notify_stop_uname="pgsr02" CRM_meta_notify_type="post" CRM_meta_on_node="pgsr01" CRM_meta_on_node_uuid="pgsr01" CRM_meta_physical_host="bl460g8n3" CRM_meta_timeout="60000" />
+ </rsc_op>
+@@ -464,7 +464,7 @@
+ </synapse>
+ <synapse id="32">
+ <action_set>
+- <rsc_op id="80" operation="notify" operation_key="pgsql_pre_notify_demote_0" internal_operation_key="pgsql:1_pre_notify_demote_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3">
++ <rsc_op id="82" operation="notify" operation_key="pgsql_pre_notify_demote_0" internal_operation_key="pgsql:1_pre_notify_demote_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3">
+ <primitive id="pgsql" long-id="pgsql:1" class="ocf" provider="heartbeat" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="pgsql:0 pgsql:1" CRM_meta_notify_active_uname="pgsr02 pgsr01" CRM_meta_notify_all_uname="bl460g8n3 bl460g8n4 pgsr01 pgsr02" CRM_meta_notify_available_uname="bl460g8n4 bl460g8n3 pgsr01 pgsr02" CRM_meta_notify_demote_resource="pgsql:0" CRM_meta_notify_demote_uname="pgsr02" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="pgsql:0 pgsql:1" CRM_meta_notify_master_uname="pgsr02 pgsr01" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="pgsql:0" CRM_meta_notify_stop_uname="pgsr02" CRM_meta_notify_type="pre" CRM_meta_on_node="pgsr01" CRM_meta_on_node_uuid="pgsr01" CRM_meta_physical_host="bl460g8n3" CRM_meta_timeout="60000" />
+ </rsc_op>
+@@ -477,7 +477,7 @@
+ </synapse>
+ <synapse id="33">
+ <action_set>
+- <rsc_op id="79" operation="notify" operation_key="pgsql_pre_notify_stop_0" internal_operation_key="pgsql:1_pre_notify_stop_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3">
++ <rsc_op id="81" operation="notify" operation_key="pgsql_pre_notify_stop_0" internal_operation_key="pgsql:1_pre_notify_stop_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3">
+ <primitive id="pgsql" long-id="pgsql:1" class="ocf" provider="heartbeat" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="pgsql:0 pgsql:1" CRM_meta_notify_active_uname="pgsr02 pgsr01" CRM_meta_notify_all_uname="bl460g8n3 bl460g8n4 pgsr01 pgsr02" CRM_meta_notify_available_uname="bl460g8n4 bl460g8n3 pgsr01 pgsr02" CRM_meta_notify_demote_resource="pgsql:0" CRM_meta_notify_demote_uname="pgsr02" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="pgsql:0 pgsql:1" CRM_meta_notify_master_uname="pgsr02 pgsr01" CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="pgsql:0" CRM_meta_notify_stop_uname="pgsr02" CRM_meta_notify_type="pre" CRM_meta_on_node="pgsr01" CRM_meta_on_node_uuid="pgsr01" CRM_meta_physical_host="bl460g8n3" CRM_meta_timeout="60000" />
+ </rsc_op>
+@@ -534,7 +534,7 @@
+ <pseudo_event id="66" operation="notify" operation_key="msPostgresql_post_notify_demoted_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="81" operation="notify" operation_key="pgsql_post_notify_demote_0" internal_operation_key="pgsql:1_post_notify_demote_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3"/>
++ <rsc_op id="83" operation="notify" operation_key="pgsql_post_notify_demote_0" internal_operation_key="pgsql:1_post_notify_demote_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -564,7 +564,7 @@
+ <pseudo_event id="64" operation="notify" operation_key="msPostgresql_pre_notify_demote_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="80" operation="notify" operation_key="pgsql_pre_notify_demote_0" internal_operation_key="pgsql:1_pre_notify_demote_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3"/>
++ <rsc_op id="82" operation="notify" operation_key="pgsql_pre_notify_demote_0" internal_operation_key="pgsql:1_pre_notify_demote_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -644,7 +644,7 @@
+ <pseudo_event id="52" operation="notify" operation_key="msPostgresql_pre_notify_stop_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="79" operation="notify" operation_key="pgsql_pre_notify_stop_0" internal_operation_key="pgsql:1_pre_notify_stop_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3"/>
++ <rsc_op id="81" operation="notify" operation_key="pgsql_pre_notify_stop_0" internal_operation_key="pgsql:1_pre_notify_stop_0" on_node="pgsr01" on_node_uuid="pgsr01" router_node="bl460g8n3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+--
+1.8.3.1
+
+
+From 6ab7a2cb1ed28548706db3bc8c006cea3b605681 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Mon, 15 Oct 2018 11:24:46 -0500
+Subject: [PATCH 9/9] Test: scheduler: order guest node connection recovery
+ after container probe
+
+---
+ pengine/regression.sh | 1 +
+ pengine/test10/guest-node-cleanup.dot | 64 ++++++
+ pengine/test10/guest-node-cleanup.exp | 319 ++++++++++++++++++++++++++++++
+ pengine/test10/guest-node-cleanup.scores | 81 ++++++++
+ pengine/test10/guest-node-cleanup.summary | 55 ++++++
+ pengine/test10/guest-node-cleanup.xml | 304 ++++++++++++++++++++++++++++
+ 6 files changed, 824 insertions(+)
+ create mode 100644 pengine/test10/guest-node-cleanup.dot
+ create mode 100644 pengine/test10/guest-node-cleanup.exp
+ create mode 100644 pengine/test10/guest-node-cleanup.scores
+ create mode 100644 pengine/test10/guest-node-cleanup.summary
+ create mode 100644 pengine/test10/guest-node-cleanup.xml
+
+diff --git a/pengine/regression.sh b/pengine/regression.sh
+index deca1b6..ead5fd8 100755
+--- a/pengine/regression.sh
++++ b/pengine/regression.sh
+@@ -854,6 +854,7 @@ do_test whitebox-migrate1 "Migrate both container and connection resource"
+ do_test whitebox-imply-stop-on-fence "imply stop action on container node rsc when host node is fenced"
+ do_test whitebox-nested-group "Verify guest remote-node works nested in a group"
+ do_test guest-node-host-dies "Verify guest node is recovered if host goes away"
++do_test guest-node-cleanup "Order guest node connection recovery after container probe"
+
+ echo ""
+ do_test remote-startup-probes "Baremetal remote-node startup probes"
+diff --git a/pengine/test10/guest-node-cleanup.dot b/pengine/test10/guest-node-cleanup.dot
+new file mode 100644
+index 0000000..45fe4be
+--- /dev/null
++++ b/pengine/test10/guest-node-cleanup.dot
+@@ -0,0 +1,64 @@
++digraph "g" {
++"all_stopped" -> "lxc1_start_0 rhel7-1" [ style = bold]
++"all_stopped" [ style=bold color="green" fontcolor="orange"]
++"container1_monitor_0 rhel7-1" -> "container1_start_0 rhel7-1" [ style = bold]
++"container1_monitor_0 rhel7-1" -> "lxc1_stop_0 rhel7-1" [ style = bold]
++"container1_monitor_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
++"container1_start_0 rhel7-1" -> "lxc-ms_promote_0 lxc1" [ style = bold]
++"container1_start_0 rhel7-1" -> "lxc-ms_start_0 lxc1" [ style = bold]
++"container1_start_0 rhel7-1" -> "lxc1_start_0 rhel7-1" [ style = bold]
++"container1_start_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
++"lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold]
++"lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc1" [ style = bold]
++"lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"]
++"lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold]
++"lxc-ms-master_demoted_0" -> "lxc-ms-master_start_0" [ style = bold]
++"lxc-ms-master_demoted_0" -> "lxc-ms-master_stop_0" [ style = bold]
++"lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc1" [ style = bold]
++"lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"]
++"lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
++"lxc-ms-master_running_0" -> "lxc-ms-master_promote_0" [ style = bold]
++"lxc-ms-master_running_0" [ style=bold color="green" fontcolor="orange"]
++"lxc-ms-master_start_0" -> "lxc-ms-master_running_0" [ style = bold]
++"lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc1" [ style = bold]
++"lxc-ms-master_start_0" [ style=bold color="green" fontcolor="orange"]
++"lxc-ms-master_stop_0" -> "lxc-ms-master_stopped_0" [ style = bold]
++"lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc1" [ style = bold]
++"lxc-ms-master_stop_0" [ style=bold color="green" fontcolor="orange"]
++"lxc-ms-master_stopped_0" -> "lxc-ms-master_promote_0" [ style = bold]
++"lxc-ms-master_stopped_0" -> "lxc-ms-master_start_0" [ style = bold]
++"lxc-ms-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"lxc-ms_demote_0 lxc1" -> "lxc-ms-master_demoted_0" [ style = bold]
++"lxc-ms_demote_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold]
++"lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold]
++"lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="orange"]
++"lxc-ms_promote_0 lxc1" -> "lxc-ms-master_promoted_0" [ style = bold]
++"lxc-ms_promote_0 lxc1" [ style=bold color="green" fontcolor="black"]
++"lxc-ms_start_0 lxc1" -> "lxc-ms-master_running_0" [ style = bold]
++"lxc-ms_start_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold]
++"lxc-ms_start_0 lxc1" [ style=bold color="green" fontcolor="black"]
++"lxc-ms_stop_0 lxc1" -> "all_stopped" [ style = bold]
++"lxc-ms_stop_0 lxc1" -> "lxc-ms-master_stopped_0" [ style = bold]
++"lxc-ms_stop_0 lxc1" -> "lxc-ms_start_0 lxc1" [ style = bold]
++"lxc-ms_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"]
++"lxc1_monitor_30000 rhel7-1" [ style=bold color="green" fontcolor="black"]
++"lxc1_start_0 rhel7-1" -> "lxc-ms_promote_0 lxc1" [ style = bold]
++"lxc1_start_0 rhel7-1" -> "lxc-ms_start_0 lxc1" [ style = bold]
++"lxc1_start_0 rhel7-1" -> "lxc1_monitor_30000 rhel7-1" [ style = bold]
++"lxc1_start_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
++"lxc1_stop_0 rhel7-1" -> "all_stopped" [ style = bold]
++"lxc1_stop_0 rhel7-1" -> "lxc1_start_0 rhel7-1" [ style = bold]
++"lxc1_stop_0 rhel7-1" -> "stonith 'reboot' lxc1" [ style = bold]
++"lxc1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
++"stonith 'reboot' lxc1" -> "lxc-ms-master_stop_0" [ style = bold]
++"stonith 'reboot' lxc1" -> "lxc-ms_demote_0 lxc1" [ style = bold]
++"stonith 'reboot' lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold]
++"stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold]
++"stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"]
++"stonith_complete" -> "all_stopped" [ style = bold]
++"stonith_complete" -> "container1_start_0 rhel7-1" [ style = bold]
++"stonith_complete" -> "lxc-ms_promote_0 lxc1" [ style = bold]
++"stonith_complete" -> "lxc-ms_start_0 lxc1" [ style = bold]
++"stonith_complete" [ style=bold color="green" fontcolor="orange"]
++}
+diff --git a/pengine/test10/guest-node-cleanup.exp b/pengine/test10/guest-node-cleanup.exp
+new file mode 100644
+index 0000000..9503a03
+--- /dev/null
++++ b/pengine/test10/guest-node-cleanup.exp
+@@ -0,0 +1,319 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="1" transition_id="0">
++ <synapse id="0">
++ <action_set>
++ <rsc_op id="11" operation="start" operation_key="container1_start_0" on_node="rhel7-1" on_node_uuid="1">
++ <primitive id="container1" class="ocf" provider="heartbeat" type="VirtualDomain"/>
++ <attributes CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_remote_node="lxc1" CRM_meta_timeout="90000" config="/var/lib/pacemaker/cts/lxc/lxc1.xml" force_stop="true" hypervisor="lxc:///"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="6" operation="monitor" operation_key="container1_monitor_0" on_node="rhel7-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="32" operation="stonith_complete" operation_key="stonith_complete"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="1">
++ <action_set>
++ <rsc_op id="6" operation="monitor" operation_key="container1_monitor_0" on_node="rhel7-1" on_node_uuid="1">
++ <primitive id="container1" class="ocf" provider="heartbeat" type="VirtualDomain"/>
++ <attributes CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_remote_node="lxc1" CRM_meta_timeout="90000" config="/var/lib/pacemaker/cts/lxc/lxc1.xml" force_stop="true" hypervisor="lxc:///"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="2">
++ <action_set>
++ <rsc_op id="19" operation="promote" operation_key="lxc-ms_promote_0" internal_operation_key="lxc-ms:1_promote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-1">
++ <primitive id="lxc-ms" long-id="lxc-ms:1" class="ocf" provider="pacemaker" type="Stateful"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="lxc1" CRM_meta_on_node_uuid="lxc1" CRM_meta_physical_host="rhel7-1" CRM_meta_timeout="90000" />
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="11" operation="start" operation_key="container1_start_0" on_node="rhel7-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="16" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:1_demote_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="18" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:1_start_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="24" operation="promote" operation_key="lxc-ms-master_promote_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="29" operation="start" operation_key="lxc1_start_0" on_node="rhel7-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="32" operation="stonith_complete" operation_key="stonith_complete"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="3">
++ <action_set>
++ <rsc_op id="18" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:1_start_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-1">
++ <primitive id="lxc-ms" long-id="lxc-ms:1" class="ocf" provider="pacemaker" type="Stateful"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="lxc1" CRM_meta_on_node_uuid="lxc1" CRM_meta_physical_host="rhel7-1" CRM_meta_timeout="90000" />
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="11" operation="start" operation_key="container1_start_0" on_node="rhel7-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="17" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:1_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="20" operation="start" operation_key="lxc-ms-master_start_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="29" operation="start" operation_key="lxc1_start_0" on_node="rhel7-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="32" operation="stonith_complete" operation_key="stonith_complete"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="4">
++ <action_set>
++ <pseudo_event id="17" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:1_stop_0">
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="90000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="16" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:1_demote_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="22" operation="stop" operation_key="lxc-ms-master_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="33" operation="stonith" operation_key="stonith-lxc1-reboot" on_node="lxc1" on_node_uuid="lxc1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="5">
++ <action_set>
++ <pseudo_event id="16" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:1_demote_0">
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="90000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="26" operation="demote" operation_key="lxc-ms-master_demote_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="33" operation="stonith" operation_key="stonith-lxc1-reboot" on_node="lxc1" on_node_uuid="lxc1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="6" priority="1000000">
++ <action_set>
++ <pseudo_event id="27" operation="demoted" operation_key="lxc-ms-master_demoted_0">
++ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="90000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="16" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:1_demote_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="26" operation="demote" operation_key="lxc-ms-master_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7">
++ <action_set>
++ <pseudo_event id="26" operation="demote" operation_key="lxc-ms-master_demote_0">
++ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="90000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="8" priority="1000000">
++ <action_set>
++ <pseudo_event id="25" operation="promoted" operation_key="lxc-ms-master_promoted_0">
++ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="90000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="19" operation="promote" operation_key="lxc-ms_promote_0" internal_operation_key="lxc-ms:1_promote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="9">
++ <action_set>
++ <pseudo_event id="24" operation="promote" operation_key="lxc-ms-master_promote_0">
++ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="90000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="21" operation="running" operation_key="lxc-ms-master_running_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="23" operation="stopped" operation_key="lxc-ms-master_stopped_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="27" operation="demoted" operation_key="lxc-ms-master_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="10" priority="1000000">
++ <action_set>
++ <pseudo_event id="23" operation="stopped" operation_key="lxc-ms-master_stopped_0">
++ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="90000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="17" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:1_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="22" operation="stop" operation_key="lxc-ms-master_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="11">
++ <action_set>
++ <pseudo_event id="22" operation="stop" operation_key="lxc-ms-master_stop_0">
++ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="90000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="27" operation="demoted" operation_key="lxc-ms-master_demoted_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="33" operation="stonith" operation_key="stonith-lxc1-reboot" on_node="lxc1" on_node_uuid="lxc1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="12" priority="1000000">
++ <action_set>
++ <pseudo_event id="21" operation="running" operation_key="lxc-ms-master_running_0">
++ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="90000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="18" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:1_start_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="20" operation="start" operation_key="lxc-ms-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="13">
++ <action_set>
++ <pseudo_event id="20" operation="start" operation_key="lxc-ms-master_start_0">
++ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="90000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="23" operation="stopped" operation_key="lxc-ms-master_stopped_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="27" operation="demoted" operation_key="lxc-ms-master_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="14">
++ <action_set>
++ <rsc_op id="29" operation="start" operation_key="lxc1_start_0" on_node="rhel7-1" on_node_uuid="1">
++ <primitive id="lxc1" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="container1" CRM_meta_name="start" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="60000" />
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="5" operation="all_stopped" operation_key="all_stopped"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="11" operation="start" operation_key="container1_start_0" on_node="rhel7-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="28" operation="stop" operation_key="lxc1_stop_0" on_node="rhel7-1" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="15">
++ <action_set>
++ <rsc_op id="28" operation="stop" operation_key="lxc1_stop_0" on_node="rhel7-1" on_node_uuid="1">
++ <primitive id="lxc1" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="container1" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="90000" />
++ <downed>
++ <node id="lxc1"/>
++ </downed>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="6" operation="monitor" operation_key="container1_monitor_0" on_node="rhel7-1" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="16">
++ <action_set>
++ <rsc_op id="2" operation="monitor" operation_key="lxc1_monitor_30000" on_node="rhel7-1" on_node_uuid="1">
++ <primitive id="lxc1" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="container1" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="30000" />
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="29" operation="start" operation_key="lxc1_start_0" on_node="rhel7-1" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="17">
++ <action_set>
++ <pseudo_event id="33" operation="stonith" operation_key="stonith-lxc1-reboot" on_node="lxc1" on_node_uuid="lxc1">
++ <attributes CRM_meta_master_lxc_ms="10" CRM_meta_on_node="lxc1" CRM_meta_on_node_uuid="lxc1" CRM_meta_stonith_action="reboot" />
++ <downed>
++ <node id="lxc1"/>
++ </downed>
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="28" operation="stop" operation_key="lxc1_stop_0" on_node="rhel7-1" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="18">
++ <action_set>
++ <pseudo_event id="32" operation="stonith_complete" operation_key="stonith_complete">
++ <attributes />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="33" operation="stonith" operation_key="stonith-lxc1-reboot" on_node="lxc1" on_node_uuid="lxc1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="19">
++ <action_set>
++ <pseudo_event id="5" operation="all_stopped" operation_key="all_stopped">
++ <attributes />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="17" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:1_stop_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="28" operation="stop" operation_key="lxc1_stop_0" on_node="rhel7-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="32" operation="stonith_complete" operation_key="stonith_complete"/>
++ </trigger>
++ </inputs>
++ </synapse>
++</transition_graph>
+diff --git a/pengine/test10/guest-node-cleanup.scores b/pengine/test10/guest-node-cleanup.scores
+new file mode 100644
+index 0000000..9bc8250
+--- /dev/null
++++ b/pengine/test10/guest-node-cleanup.scores
+@@ -0,0 +1,81 @@
++Allocation scores:
++Using the original execution date of: 2018-10-15 16:02:04Z
++clone_color: lxc-ms-master allocation score on lxc1: INFINITY
++clone_color: lxc-ms-master allocation score on lxc2: INFINITY
++clone_color: lxc-ms-master allocation score on rhel7-1: 0
++clone_color: lxc-ms-master allocation score on rhel7-2: 0
++clone_color: lxc-ms-master allocation score on rhel7-3: 0
++clone_color: lxc-ms-master allocation score on rhel7-4: 0
++clone_color: lxc-ms-master allocation score on rhel7-5: 0
++clone_color: lxc-ms:0 allocation score on lxc1: INFINITY
++clone_color: lxc-ms:0 allocation score on lxc2: INFINITY
++clone_color: lxc-ms:0 allocation score on rhel7-1: 0
++clone_color: lxc-ms:0 allocation score on rhel7-2: 0
++clone_color: lxc-ms:0 allocation score on rhel7-3: 0
++clone_color: lxc-ms:0 allocation score on rhel7-4: 0
++clone_color: lxc-ms:0 allocation score on rhel7-5: 0
++clone_color: lxc-ms:1 allocation score on lxc1: INFINITY
++clone_color: lxc-ms:1 allocation score on lxc2: INFINITY
++clone_color: lxc-ms:1 allocation score on rhel7-1: 0
++clone_color: lxc-ms:1 allocation score on rhel7-2: 0
++clone_color: lxc-ms:1 allocation score on rhel7-3: 0
++clone_color: lxc-ms:1 allocation score on rhel7-4: 0
++clone_color: lxc-ms:1 allocation score on rhel7-5: 0
++lxc-ms:0 promotion score on lxc2: INFINITY
++lxc-ms:1 promotion score on lxc1: INFINITY
++native_color: Fencing allocation score on lxc1: -INFINITY
++native_color: Fencing allocation score on lxc2: -INFINITY
++native_color: Fencing allocation score on rhel7-1: 0
++native_color: Fencing allocation score on rhel7-2: 0
++native_color: Fencing allocation score on rhel7-3: 0
++native_color: Fencing allocation score on rhel7-4: 0
++native_color: Fencing allocation score on rhel7-5: 0
++native_color: FencingPass allocation score on lxc1: -INFINITY
++native_color: FencingPass allocation score on lxc2: -INFINITY
++native_color: FencingPass allocation score on rhel7-1: 0
++native_color: FencingPass allocation score on rhel7-2: 0
++native_color: FencingPass allocation score on rhel7-3: 0
++native_color: FencingPass allocation score on rhel7-4: 0
++native_color: FencingPass allocation score on rhel7-5: 0
++native_color: container1 allocation score on lxc1: -INFINITY
++native_color: container1 allocation score on lxc2: -INFINITY
++native_color: container1 allocation score on rhel7-1: INFINITY
++native_color: container1 allocation score on rhel7-2: 0
++native_color: container1 allocation score on rhel7-3: 0
++native_color: container1 allocation score on rhel7-4: 0
++native_color: container1 allocation score on rhel7-5: 0
++native_color: container2 allocation score on lxc1: -INFINITY
++native_color: container2 allocation score on lxc2: -INFINITY
++native_color: container2 allocation score on rhel7-1: INFINITY
++native_color: container2 allocation score on rhel7-2: 0
++native_color: container2 allocation score on rhel7-3: 0
++native_color: container2 allocation score on rhel7-4: 0
++native_color: container2 allocation score on rhel7-5: 0
++native_color: lxc-ms:0 allocation score on lxc1: INFINITY
++native_color: lxc-ms:0 allocation score on lxc2: INFINITY
++native_color: lxc-ms:0 allocation score on rhel7-1: 0
++native_color: lxc-ms:0 allocation score on rhel7-2: 0
++native_color: lxc-ms:0 allocation score on rhel7-3: 0
++native_color: lxc-ms:0 allocation score on rhel7-4: 0
++native_color: lxc-ms:0 allocation score on rhel7-5: 0
++native_color: lxc-ms:1 allocation score on lxc1: INFINITY
++native_color: lxc-ms:1 allocation score on lxc2: -INFINITY
++native_color: lxc-ms:1 allocation score on rhel7-1: 0
++native_color: lxc-ms:1 allocation score on rhel7-2: 0
++native_color: lxc-ms:1 allocation score on rhel7-3: 0
++native_color: lxc-ms:1 allocation score on rhel7-4: 0
++native_color: lxc-ms:1 allocation score on rhel7-5: 0
++native_color: lxc1 allocation score on lxc1: -INFINITY
++native_color: lxc1 allocation score on lxc2: -INFINITY
++native_color: lxc1 allocation score on rhel7-1: 0
++native_color: lxc1 allocation score on rhel7-2: -INFINITY
++native_color: lxc1 allocation score on rhel7-3: -INFINITY
++native_color: lxc1 allocation score on rhel7-4: -INFINITY
++native_color: lxc1 allocation score on rhel7-5: -INFINITY
++native_color: lxc2 allocation score on lxc1: -INFINITY
++native_color: lxc2 allocation score on lxc2: -INFINITY
++native_color: lxc2 allocation score on rhel7-1: 0
++native_color: lxc2 allocation score on rhel7-2: -INFINITY
++native_color: lxc2 allocation score on rhel7-3: -INFINITY
++native_color: lxc2 allocation score on rhel7-4: -INFINITY
++native_color: lxc2 allocation score on rhel7-5: -INFINITY
+diff --git a/pengine/test10/guest-node-cleanup.summary b/pengine/test10/guest-node-cleanup.summary
+new file mode 100644
+index 0000000..6378f48
+--- /dev/null
++++ b/pengine/test10/guest-node-cleanup.summary
+@@ -0,0 +1,55 @@
++Using the original execution date of: 2018-10-15 16:02:04Z
++
++Current cluster status:
++Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
++Containers: [ lxc2:container2 ]
++
++ Fencing (stonith:fence_xvm): Started rhel7-2
++ FencingPass (stonith:fence_dummy): Started rhel7-3
++ container1 (ocf::heartbeat:VirtualDomain): FAILED
++ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-1
++ Master/Slave Set: lxc-ms-master [lxc-ms]
++ Slaves: [ lxc2 ]
++ Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
++
++Transition Summary:
++ * Fence (reboot) lxc1 (resource: container1) 'guest is unclean'
++ * Start container1 ( rhel7-1 )
++ * Recover lxc-ms:1 ( Master lxc1 )
++ * Restart lxc1 ( rhel7-1 ) due to required container1 start
++
++Executing cluster transition:
++ * Resource action: container1 monitor on rhel7-1
++ * Pseudo action: lxc-ms-master_demote_0
++ * Resource action: lxc1 stop on rhel7-1
++ * Pseudo action: stonith-lxc1-reboot on lxc1
++ * Pseudo action: stonith_complete
++ * Resource action: container1 start on rhel7-1
++ * Pseudo action: lxc-ms_demote_0
++ * Pseudo action: lxc-ms-master_demoted_0
++ * Pseudo action: lxc-ms-master_stop_0
++ * Pseudo action: lxc-ms_stop_0
++ * Pseudo action: lxc-ms-master_stopped_0
++ * Pseudo action: lxc-ms-master_start_0
++ * Pseudo action: all_stopped
++ * Resource action: lxc1 start on rhel7-1
++ * Resource action: lxc1 monitor=30000 on rhel7-1
++ * Resource action: lxc-ms start on lxc1
++ * Pseudo action: lxc-ms-master_running_0
++ * Pseudo action: lxc-ms-master_promote_0
++ * Resource action: lxc-ms promote on lxc1
++ * Pseudo action: lxc-ms-master_promoted_0
++Using the original execution date of: 2018-10-15 16:02:04Z
++
++Revised cluster status:
++Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
++Containers: [ lxc1:container1 lxc2:container2 ]
++
++ Fencing (stonith:fence_xvm): Started rhel7-2
++ FencingPass (stonith:fence_dummy): Started rhel7-3
++ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-1
++ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-1
++ Master/Slave Set: lxc-ms-master [lxc-ms]
++ Masters: [ lxc1 ]
++ Slaves: [ lxc2 ]
++
+diff --git a/pengine/test10/guest-node-cleanup.xml b/pengine/test10/guest-node-cleanup.xml
+new file mode 100644
+index 0000000..35835bc
+--- /dev/null
++++ b/pengine/test10/guest-node-cleanup.xml
+@@ -0,0 +1,304 @@
++<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.9" epoch="238" num_updates="0" admin_epoch="0" cib-last-written="Mon Oct 15 11:02:03 2018" update-origin="rhel7-1" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2" execution-date="1539619324">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/>
++ <nvpair id="cts-start-failure-is-fatal" name="start-failure-is-fatal" value="false"/>
++ <nvpair id="cts-pe-input-series-max" name="pe-input-series-max" value="5000"/>
++ <nvpair id="cts-shutdown-escalation" name="shutdown-escalation" value="5min"/>
++ <nvpair id="cts-batch-limit" name="batch-limit" value="10"/>
++ <nvpair id="cts-dc-deadtime" name="dc-deadtime" value="5s"/>
++ <nvpair id="cts-no-quorum-policy" name="no-quorum-policy" value="stop"/>
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="mycluster"/>
++ <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1539619323"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="rhel7-1"/>
++ <node id="2" uname="rhel7-2"/>
++ <node id="3" uname="rhel7-3">
++ <instance_attributes id="nodes-3">
++ <nvpair id="nodes-3-standby" name="standby" value="off"/>
++ </instance_attributes>
++ </node>
++ <node id="4" uname="rhel7-4"/>
++ <node id="5" uname="rhel7-5">
++ <instance_attributes id="nodes-5">
++ <nvpair id="nodes-5-standby" name="standby" value="off"/>
++ </instance_attributes>
++ </node>
++ </nodes>
++ <resources>
++ <primitive class="stonith" id="Fencing" type="fence_xvm">
++ <meta_attributes id="Fencing-meta">
++ <nvpair id="Fencing-migration-threshold" name="migration-threshold" value="5"/>
++ </meta_attributes>
++ <instance_attributes id="Fencing-params">
++ <nvpair id="Fencing-key_file" name="key_file" value="/etc/pacemaker/fence_xvm.key"/>
++ <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/>
++ <nvpair id="Fencing-pcmk_host_map" name="pcmk_host_map" value="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
++ <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 rhel7-3 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5"/>
++ </instance_attributes>
++ <operations>
++ <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/>
++ <op id="Fencing-stop-0" interval="0" name="stop" timeout="60s"/>
++ <op id="Fencing-start-0" interval="0" name="start" timeout="60s"/>
++ </operations>
++ </primitive>
++ <primitive class="stonith" id="FencingPass" type="fence_dummy">
++ <instance_attributes id="FencingPass-params">
++ <nvpair id="FencingPass-random_sleep_range" name="random_sleep_range" value="30"/>
++ <nvpair id="FencingPass-pcmk_host_list" name="pcmk_host_list" value="rhel7-4 remote-rhel7-4"/>
++ <nvpair id="FencingPass-mode" name="mode" value="pass"/>
++ </instance_attributes>
++ </primitive>
++ <primitive class="ocf" id="container1" provider="heartbeat" type="VirtualDomain">
++ <instance_attributes id="container1-instance_attributes">
++ <nvpair id="container1-instance_attributes-force_stop" name="force_stop" value="true"/>
++ <nvpair id="container1-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
++ <nvpair id="container1-instance_attributes-config" name="config" value="/var/lib/pacemaker/cts/lxc/lxc1.xml"/>
++ </instance_attributes>
++ <utilization id="container1-utilization">
++ <nvpair id="container1-utilization-cpu" name="cpu" value="1"/>
++ <nvpair id="container1-utilization-hv_memory" name="hv_memory" value="196"/>
++ </utilization>
++ <meta_attributes id="container1-meta_attributes">
++ <nvpair id="container1-meta_attributes-remote-node" name="remote-node" value="lxc1"/>
++ </meta_attributes>
++ </primitive>
++ <primitive class="ocf" id="container2" provider="heartbeat" type="VirtualDomain">
++ <instance_attributes id="container2-instance_attributes">
++ <nvpair id="container2-instance_attributes-force_stop" name="force_stop" value="true"/>
++ <nvpair id="container2-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
++ <nvpair id="container2-instance_attributes-config" name="config" value="/var/lib/pacemaker/cts/lxc/lxc2.xml"/>
++ </instance_attributes>
++ <utilization id="container2-utilization">
++ <nvpair id="container2-utilization-cpu" name="cpu" value="1"/>
++ <nvpair id="container2-utilization-hv_memory" name="hv_memory" value="196"/>
++ </utilization>
++ <meta_attributes id="container2-meta_attributes">
++ <nvpair id="container2-meta_attributes-remote-node" name="remote-node" value="lxc2"/>
++ </meta_attributes>
++ </primitive>
++ <master id="lxc-ms-master">
++ <primitive class="ocf" id="lxc-ms" provider="pacemaker" type="Stateful">
++ <instance_attributes id="lxc-ms-instance_attributes"/>
++ <operations>
++ <op id="lxc-ms-monitor-interval-10s" interval="10s" name="monitor"/>
++ </operations>
++ </primitive>
++ <meta_attributes id="lxc-ms-meta_attributes">
++ <nvpair id="lxc-ms-meta_attributes-master-max" name="master-max" value="1"/>
++ <nvpair id="lxc-ms-meta_attributes-clone-max" name="clone-max" value="2"/>
++ </meta_attributes>
++ </master>
++ </resources>
++ <constraints>
++ <rsc_location id="cli-prefer-container1" rsc="container1" role="Started" node="rhel7-1" score="INFINITY"/>
++ <rsc_location id="cli-prefer-container2" rsc="container2" role="Started" node="rhel7-1" score="INFINITY"/>
++ <rsc_location id="lxc-ms-location-lxc1" node="lxc1" rsc="lxc-ms-master" score="INFINITY"/>
++ <rsc_location id="lxc-ms-location-lxc2" node="lxc2" rsc="lxc-ms-master" score="INFINITY"/>
++ </constraints>
++ <fencing-topology>
++ <fencing-level devices="FencingPass,Fencing" id="cts-rhel7-4.1" index="1" target="rhel7-4"/>
++ <fencing-level devices="FencingPass,Fencing" id="cts-remote-rhel7-4.1" index="1" target="remote-rhel7-4"/>
++ </fencing-topology>
++ <op_defaults>
++ <meta_attributes id="cts-op_defaults-meta">
++ <nvpair id="cts-op_defaults-timeout" name="timeout" value="90s"/>
++ </meta_attributes>
++ </op_defaults>
++ <alerts>
++ <alert id="alert-1" path="/var/lib/pacemaker/notify.sh">
++ <recipient id="alert-1-recipient-1" value="/run/crm/alert.log"/>
++ </alert>
++ </alerts>
++ <rsc_defaults>
++ <meta_attributes id="rsc_defaults-options"/>
++ </rsc_defaults>
++ </configuration>
++ <status>
++ <node_state id="5" uname="rhel7-5" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
++ <lrm id="5">
++ <lrm_resources>
++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="58:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;58:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-5" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1539618536" last-rc-change="1539618536" exec-time="58" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
++ </lrm_resource>
++ <lrm_resource id="FencingPass" type="fence_dummy" class="stonith">
++ <lrm_rsc_op id="FencingPass_last_0" operation_key="FencingPass_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="59:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;59:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-5" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1539618538" last-rc-change="1539618538" exec-time="0" queue-time="0" op-digest="eb5a03cf2da3e1de4e70b23956bdd8b4"/>
++ </lrm_resource>
++ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="19:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;19:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-5" call-id="107" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="273" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
++ </lrm_resource>
++ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="20:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;20:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-5" call-id="111" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="284" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
++ </lrm_resource>
++ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
++ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="21:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;21:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-5" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
++ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="14:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;14:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-5" call-id="117" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="35" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
++ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="15:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;15:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-5" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="5">
++ <instance_attributes id="status-5"/>
++ </transient_attributes>
++ </node_state>
++ <node_state id="2" uname="rhel7-2" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
++ <lrm id="2">
++ <lrm_resources>
++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="24:57:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;24:57:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-2" call-id="105" rc-code="0" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="93" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
++ <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="25:57:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;25:57:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-2" call-id="107" rc-code="0" op-status="0" interval="120000" last-rc-change="1539618980" exec-time="71" queue-time="0" op-digest="cb34bc19df153021ce8f301baa293f35"/>
++ </lrm_resource>
++ <lrm_resource id="FencingPass" type="fence_dummy" class="stonith">
++ <lrm_rsc_op id="FencingPass_last_0" operation_key="FencingPass_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="26:57:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;26:57:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-2" call-id="99" rc-code="0" op-status="0" interval="0" last-run="1539618979" last-rc-change="1539618979" exec-time="1" queue-time="0" op-digest="eb5a03cf2da3e1de4e70b23956bdd8b4"/>
++ </lrm_resource>
++ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="7:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;7:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-2" call-id="103" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="481" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
++ </lrm_resource>
++ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="8:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;8:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-2" call-id="112" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="328" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
++ </lrm_resource>
++ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
++ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="6:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;6:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-2" call-id="117" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="108" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
++ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="7:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;7:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-2" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
++ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="8:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;8:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-2" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="2">
++ <instance_attributes id="status-2"/>
++ </transient_attributes>
++ </node_state>
++ <node_state id="3" uname="rhel7-3" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
++ <lrm id="3">
++ <lrm_resources>
++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="30:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;30:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1539618536" last-rc-change="1539618536" exec-time="13" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
++ </lrm_resource>
++ <lrm_resource id="FencingPass" type="fence_dummy" class="stonith">
++ <lrm_rsc_op id="FencingPass_last_0" operation_key="FencingPass_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="18:58:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;18:58:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-3" call-id="112" rc-code="0" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="9073" queue-time="0" op-digest="eb5a03cf2da3e1de4e70b23956bdd8b4"/>
++ </lrm_resource>
++ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="11:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;11:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-3" call-id="107" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="302" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
++ </lrm_resource>
++ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="12:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;12:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-3" call-id="111" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="306" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
++ </lrm_resource>
++ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
++ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="9:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;9:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-3" call-id="117" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="45" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
++ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="10:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;10:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-3" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
++ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="11:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;11:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-3" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="3">
++ <instance_attributes id="status-3"/>
++ </transient_attributes>
++ </node_state>
++ <node_state id="4" uname="rhel7-4" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
++ <lrm id="4">
++ <lrm_resources>
++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="44:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;44:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-4" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1539618536" last-rc-change="1539618536" exec-time="54" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
++ </lrm_resource>
++ <lrm_resource id="FencingPass" type="fence_dummy" class="stonith">
++ <lrm_rsc_op id="FencingPass_last_0" operation_key="FencingPass_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="45:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;45:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-4" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1539618538" last-rc-change="1539618538" exec-time="0" queue-time="0" op-digest="eb5a03cf2da3e1de4e70b23956bdd8b4"/>
++ </lrm_resource>
++ <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="15:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;15:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-4" call-id="99" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="277" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
++ </lrm_resource>
++ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="16:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;16:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-4" call-id="103" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="309" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
++ </lrm_resource>
++ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
++ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="17:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;17:57:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-4" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
++ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="12:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;12:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-4" call-id="109" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="30" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
++ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="13:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;13:58:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-4" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="4">
++ <instance_attributes id="status-4"/>
++ </transient_attributes>
++ </node_state>
++ <node_state id="1" uname="rhel7-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="1">
++ <lrm_resources>
++ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
++ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="23:57:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;23:57:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="101" rc-code="0" op-status="0" interval="0" last-run="1539618979" last-rc-change="1539618979" exec-time="1" queue-time="0" op-digest="c7e1af5a2f7b98510353dc9f9edfef70"/>
++ <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="73:0:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;73:0:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="8" rc-code="0" op-status="0" interval="120000" last-rc-change="1539618537" exec-time="31" queue-time="0" op-digest="cb34bc19df153021ce8f301baa293f35"/>
++ </lrm_resource>
++ <lrm_resource id="FencingPass" type="fence_dummy" class="stonith">
++ <lrm_rsc_op id="FencingPass_last_0" operation_key="FencingPass_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="3:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;3:0:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="13" rc-code="7" op-status="0" interval="0" last-run="1539618538" last-rc-change="1539618538" exec-time="0" queue-time="0" op-digest="eb5a03cf2da3e1de4e70b23956bdd8b4"/>
++ </lrm_resource>
++ <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="container2_last_0" operation_key="container2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="20:58:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;20:58:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="113" rc-code="0" op-status="0" interval="0" last-run="1539618980" last-rc-change="1539618980" exec-time="1163" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
++ </lrm_resource>
++ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
++ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="5:59:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:7;5:59:7:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="120" rc-code="7" op-status="0" interval="0" last-run="1539618989" last-rc-change="1539618989" exec-time="48" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
++ <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="28:59:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;28:59:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1539618989" last-rc-change="1539618989" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="lxc1_monitor_30000" operation_key="lxc1_monitor_30000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="27:60:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;27:60:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="5" rc-code="0" op-status="0" interval="30000" last-rc-change="1539618991" exec-time="0" queue-time="0" op-digest="02a5bcf940fc8d3239701acb11438d6a"/>
++ </lrm_resource>
++ <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
++ <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="30:59:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;30:59:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="4" rc-code="0" op-status="0" interval="0" last-run="1539618989" last-rc-change="1539618989" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="lxc2_monitor_30000" operation_key="lxc2_monitor_30000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="30:60:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;30:60:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="6" rc-code="0" op-status="0" interval="30000" last-rc-change="1539618991" exec-time="0" queue-time="0" op-digest="02a5bcf940fc8d3239701acb11438d6a"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="1">
++ <instance_attributes id="status-1"/>
++ </transient_attributes>
++ </node_state>
++ <node_state remote_node="true" id="lxc1" uname="lxc1" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <lrm id="lxc1">
++ <lrm_resources>
++ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
++ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="15:61:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;15:61:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="16" rc-code="0" op-status="0" interval="0" last-run="1539618992" last-rc-change="1539618992" exec-time="166" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="lxc1">
++ <instance_attributes id="status-lxc1">
++ <nvpair id="status-lxc1-master-lxc-ms" name="master-lxc-ms" value="10"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ <node_state remote_node="true" id="lxc2" uname="lxc2" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <lrm id="lxc2">
++ <lrm_resources>
++ <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
++ <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="15:60:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;15:60:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="8" rc-code="0" op-status="0" interval="0" last-run="1539618991" last-rc-change="1539618991" exec-time="224" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="lxc-ms_monitor_10000" operation_key="lxc-ms_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="18:61:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" transition-magic="0:0;18:61:0:c4b6e92c-170d-4c6d-be0b-cc468fe47f19" exit-reason="" on_node="rhel7-1" call-id="16" rc-code="0" op-status="0" interval="10000" last-rc-change="1539618991" exec-time="19" queue-time="0" op-digest="8f6a313464b7f9e3a31cb448458b700e"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="lxc2">
++ <instance_attributes id="status-lxc2">
++ <nvpair id="status-lxc2-master-lxc-ms" name="master-lxc-ms" value="5"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ </status>
++</cib>
+--
+1.8.3.1
+
diff --git a/SOURCES/009-sbd-guest.patch b/SOURCES/009-sbd-guest.patch
new file mode 100644
index 0000000..0fd41b1
--- /dev/null
+++ b/SOURCES/009-sbd-guest.patch
@@ -0,0 +1,152 @@
+From 6f7ce4e903eed136cc9038952f9d57d4423736dd Mon Sep 17 00:00:00 2001
+From: Klaus Wenninger <klaus.wenninger@aon.at>
+Date: Sun, 4 Nov 2018 23:15:58 +0100
+Subject: [PATCH 1/2] Refactor: remote_ra: have attribute strings in msg_xml.h
+
+---
+ crmd/remote_lrmd_ra.c | 6 +++---
+ include/crm/msg_xml.h | 3 +++
+ 2 files changed, 6 insertions(+), 3 deletions(-)
+
+diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c
+index 3cdc7f0..a164fc1 100644
+--- a/crmd/remote_lrmd_ra.c
++++ b/crmd/remote_lrmd_ra.c
+@@ -724,10 +724,10 @@ handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeo
+ int timeout_used = timeout_ms > MAX_START_TIMEOUT_MS ? MAX_START_TIMEOUT_MS : timeout_ms;
+
+ for (tmp = cmd->params; tmp; tmp = tmp->next) {
+- if (safe_str_eq(tmp->key, "addr") || safe_str_eq(tmp->key, "server")) {
++ if (safe_str_eq(tmp->key, XML_RSC_ATTR_REMOTE_RA_ADDR) ||
++ safe_str_eq(tmp->key, XML_RSC_ATTR_REMOTE_RA_SERVER)) {
+ server = tmp->value;
+- }
+- if (safe_str_eq(tmp->key, "port")) {
++ } else if (safe_str_eq(tmp->key, XML_RSC_ATTR_REMOTE_RA_PORT)) {
+ port = atoi(tmp->value);
+ }
+ }
+diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h
+index 1ac4302..55f42c4 100644
+--- a/include/crm/msg_xml.h
++++ b/include/crm/msg_xml.h
+@@ -233,6 +233,9 @@
+ # define XML_RSC_ATTR_REMOTE_NODE "remote-node"
+ # define XML_RSC_ATTR_CLEAR_OP "clear_failure_op"
+ # define XML_RSC_ATTR_CLEAR_INTERVAL "clear_failure_interval"
++# define XML_RSC_ATTR_REMOTE_RA_ADDR "addr"
++# define XML_RSC_ATTR_REMOTE_RA_SERVER "server"
++# define XML_RSC_ATTR_REMOTE_RA_PORT "port"
+
+ # define XML_REMOTE_ATTR_RECONNECT_INTERVAL "reconnect_interval"
+
+--
+1.8.3.1
+
+
+From 4dae6746002b034868feda763a85de85e08834e7 Mon Sep 17 00:00:00 2001
+From: Klaus Wenninger <klaus.wenninger@aon.at>
+Date: Sun, 4 Nov 2018 23:54:11 +0100
+Subject: [PATCH 2/2] Fix: pacemaker-remote: skip remote_config_check for
+ guest-nodes
+
+This is crucial when watchdog-fencing is enabled as the sbd-check
+done by pacemaker-remote would fail on guest-containers & bundles
+(eventually tearing down pacemaker-remote inside the container)
+and even on system-virtualized-guests the sbd-check doesn't make
+sense as these guests would be fenced by stop/start-cycling the
+VM.
+---
+ crmd/crmd_lrm.h | 1 +
+ crmd/lrm_state.c | 14 +++++++++-----
+ crmd/remote_lrmd_ra.c | 20 ++++++++++++++++++++
+ 3 files changed, 30 insertions(+), 5 deletions(-)
+
+diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h
+index 7a74f7a..d115056 100644
+--- a/crmd/crmd_lrm.h
++++ b/crmd/crmd_lrm.h
+@@ -168,5 +168,6 @@ void remote_ra_fail(const char *node_name);
+ void remote_ra_process_pseudo(xmlNode *xml);
+ gboolean remote_ra_is_in_maintenance(lrm_state_t * lrm_state);
+ void remote_ra_process_maintenance_nodes(xmlNode *xml);
++gboolean remote_ra_controlling_guest(lrm_state_t * lrm_state);
+
+ gboolean process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurring_op_s *pending);
+diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c
+index 497afe1..40da762 100644
+--- a/crmd/lrm_state.c
++++ b/crmd/lrm_state.c
+@@ -500,11 +500,15 @@ crmd_remote_proxy_cb(lrmd_t *lrmd, void *userdata, xmlNode *msg)
+ const char *channel = crm_element_value(msg, F_LRMD_IPC_IPC_SERVER);
+
+ proxy = crmd_remote_proxy_new(lrmd, lrm_state->node_name, session, channel);
+- if (proxy != NULL) {
+- /* Look up stonith-watchdog-timeout and send to the remote peer for validation */
+- int rc = fsa_cib_conn->cmds->query(fsa_cib_conn, XML_CIB_TAG_CRMCONFIG, NULL, cib_scope_local);
+- fsa_cib_conn->cmds->register_callback_full(fsa_cib_conn, rc, 10, FALSE, lrmd,
+- "remote_config_check", remote_config_check, NULL);
++ if (!remote_ra_controlling_guest(lrm_state)) {
++ if (proxy != NULL) {
++ /* Look up stonith-watchdog-timeout and send to the remote peer for validation */
++ int rc = fsa_cib_conn->cmds->query(fsa_cib_conn, XML_CIB_TAG_CRMCONFIG, NULL, cib_scope_local);
++ fsa_cib_conn->cmds->register_callback_full(fsa_cib_conn, rc, 10, FALSE, lrmd,
++ "remote_config_check", remote_config_check, NULL);
++ }
++ } else {
++ crm_debug("Skipping remote_config_check for guest-nodes");
+ }
+
+ } else if (safe_str_eq(op, LRMD_IPC_OP_SHUTDOWN_REQ)) {
+diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c
+index a164fc1..0414709 100644
+--- a/crmd/remote_lrmd_ra.c
++++ b/crmd/remote_lrmd_ra.c
+@@ -85,6 +85,15 @@ typedef struct remote_ra_data_s {
+ * so we have it signalled back with the
+ * transition from pengine
+ */
++ gboolean controlling_guest; /* Similar for if we are controlling a guest
++ * or a bare-metal remote.
++ * Fortunately there is a meta-attribute in
++ * the transition already and as the
++ * situation doesn't change over time we can
++ * use the resource start for noting down
++ * the information for later use when the
++ * attributes aren't at hand.
++ */
+ } remote_ra_data_t;
+
+ static int handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms);
+@@ -721,6 +730,7 @@ handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeo
+ const char *server = NULL;
+ lrmd_key_value_t *tmp = NULL;
+ int port = 0;
++ remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
+ int timeout_used = timeout_ms > MAX_START_TIMEOUT_MS ? MAX_START_TIMEOUT_MS : timeout_ms;
+
+ for (tmp = cmd->params; tmp; tmp = tmp->next) {
+@@ -729,6 +739,8 @@ handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeo
+ server = tmp->value;
+ } else if (safe_str_eq(tmp->key, XML_RSC_ATTR_REMOTE_RA_PORT)) {
+ port = atoi(tmp->value);
++ } else if (safe_str_eq(tmp->key, CRM_META"_"XML_RSC_ATTR_CONTAINER)) {
++ ra_data->controlling_guest = TRUE;
+ }
+ }
+
+@@ -1262,3 +1274,11 @@ remote_ra_is_in_maintenance(lrm_state_t * lrm_state)
+
+ return ra_data->is_maintenance;
+ }
++
++gboolean
++remote_ra_controlling_guest(lrm_state_t * lrm_state)
++{
++ remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
++
++ return ra_data->controlling_guest;
++}
+--
+1.8.3.1
+
diff --git a/SOURCES/010-route-notify.patch b/SOURCES/010-route-notify.patch
new file mode 100644
index 0000000..c435db1
--- /dev/null
+++ b/SOURCES/010-route-notify.patch
@@ -0,0 +1,1621 @@
+From 2ba3fffcac2f11d12795a69eb019d961a0f39450 Mon Sep 17 00:00:00 2001
+From: Andrew Beekhof <andrew@beekhof.net>
+Date: Thu, 1 Nov 2018 22:08:22 +1100
+Subject: [PATCH 1/2] Fix: rhbz#1644076 - Ensure the bundle stop/demote
+ notifications are directed to the correct host
+
+---
+ pengine/graph.c | 15 ++++++++++-----
+ 1 file changed, 10 insertions(+), 5 deletions(-)
+
+diff --git a/pengine/graph.c b/pengine/graph.c
+index 236b278..9edd1a1 100644
+--- a/pengine/graph.c
++++ b/pengine/graph.c
+@@ -784,8 +784,9 @@ get_router_node(action_t *action)
+ node_t *ended_on = NULL;
+ node_t *router_node = NULL;
+ bool partial_migration = FALSE;
++ const char *task = action->task;
+
+- if (safe_str_eq(action->task, CRM_OP_FENCE) || is_remote_node(action->node) == FALSE) {
++ if (safe_str_eq(task, CRM_OP_FENCE) || is_remote_node(action->node) == FALSE) {
+ return NULL;
+ }
+
+@@ -831,11 +832,15 @@ get_router_node(action_t *action)
+ * moving to.
+ */
+
++ if (safe_str_eq(task, "notify")) {
++ task = g_hash_table_lookup(action->meta, "notify_operation");
++ }
++
+ /* 1. before connection rsc moves. */
+- if ((safe_str_eq(action->task, "stop") ||
+- safe_str_eq(action->task, "demote") ||
+- safe_str_eq(action->task, "migrate_from") ||
+- safe_str_eq(action->task, "migrate_to")) && !partial_migration) {
++ if ((safe_str_eq(task, "stop") ||
++ safe_str_eq(task, "demote") ||
++ safe_str_eq(task, "migrate_from") ||
++ safe_str_eq(task, "migrate_to")) && !partial_migration) {
+
+ router_node = began_on;
+
+--
+1.8.3.1
+
+
+From 61f9622dfd42f7d4eb71cdc2eeae374e589d34e0 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Mon, 5 Nov 2018 11:41:12 -0600
+Subject: [PATCH 2/2] Test: scheduler: ensure remote notifications are directed
+ to correct host
+
+This test ensures that the pre-notify actions sent to rabbitmq-bundle-0 for the
+stop of rabbitmq-bundle (which is moving from controller-0 to controller-1) are
+routed through controller-0.
+---
+ pengine/regression.sh | 1 +
+ pengine/test10/route-remote-notify.dot | 121 ++++++
+ pengine/test10/route-remote-notify.exp | 612 +++++++++++++++++++++++++++++
+ pengine/test10/route-remote-notify.scores | 182 +++++++++
+ pengine/test10/route-remote-notify.summary | 102 +++++
+ pengine/test10/route-remote-notify.xml | 487 +++++++++++++++++++++++
+ 6 files changed, 1505 insertions(+)
+ create mode 100644 pengine/test10/route-remote-notify.dot
+ create mode 100644 pengine/test10/route-remote-notify.exp
+ create mode 100644 pengine/test10/route-remote-notify.scores
+ create mode 100644 pengine/test10/route-remote-notify.summary
+ create mode 100644 pengine/test10/route-remote-notify.xml
+
+diff --git a/pengine/regression.sh b/pengine/regression.sh
+index ead5fd8..4ba4671 100755
+--- a/pengine/regression.sh
++++ b/pengine/regression.sh
+@@ -483,6 +483,7 @@ do_test notify-2 "Notify simple, confirm"
+ do_test notify-3 "Notify move, confirm"
+ do_test novell-239079 "Notification priority"
+ #do_test notify-2 "Notify - 764"
++do_test route-remote-notify "Route remote notify actions through correct cluster node"
+
+ echo ""
+ do_test 594 "OSDL #594 - Unrunnable actions scheduled in transition"
+diff --git a/pengine/test10/route-remote-notify.dot b/pengine/test10/route-remote-notify.dot
+new file mode 100644
+index 0000000..c473321
+--- /dev/null
++++ b/pengine/test10/route-remote-notify.dot
+@@ -0,0 +1,121 @@
++digraph "g" {
++"Cancel rabbitmq-bundle-1_monitor_30000 controller-1" [ style=bold color="green" fontcolor="black"]
++"Cancel rabbitmq-bundle-2_monitor_30000 controller-2" [ style=bold color="green" fontcolor="black"]
++"all_stopped" [ style=bold color="green" fontcolor="orange"]
++"do_shutdown controller-0" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold]
++"haproxy-bundle-docker-0_stop_0 controller-0" -> "do_shutdown controller-0" [ style = bold]
++"haproxy-bundle-docker-0_stop_0 controller-0" -> "haproxy-bundle_stopped_0" [ style = bold]
++"haproxy-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle_stop_0" -> "haproxy-bundle-docker-0_stop_0 controller-0" [ style = bold]
++"haproxy-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"haproxy-bundle_stopped_0" -> "ip-172.17.1.11_stop_0 controller-0" [ style = bold]
++"haproxy-bundle_stopped_0" -> "ip-192.168.24.12_stop_0 controller-0" [ style = bold]
++"haproxy-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"ip-172.17.1.11_monitor_10000 controller-1" [ style=bold color="green" fontcolor="black"]
++"ip-172.17.1.11_start_0 controller-1" -> "ip-172.17.1.11_monitor_10000 controller-1" [ style = bold]
++"ip-172.17.1.11_start_0 controller-1" [ style=bold color="green" fontcolor="black"]
++"ip-172.17.1.11_stop_0 controller-0" -> "all_stopped" [ style = bold]
++"ip-172.17.1.11_stop_0 controller-0" -> "do_shutdown controller-0" [ style = bold]
++"ip-172.17.1.11_stop_0 controller-0" -> "ip-172.17.1.11_start_0 controller-1" [ style = bold]
++"ip-172.17.1.11_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.24.12_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.24.12_start_0 controller-2" -> "ip-192.168.24.12_monitor_10000 controller-2" [ style = bold]
++"ip-192.168.24.12_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.24.12_stop_0 controller-0" -> "all_stopped" [ style = bold]
++"ip-192.168.24.12_stop_0 controller-0" -> "do_shutdown controller-0" [ style = bold]
++"ip-192.168.24.12_stop_0 controller-0" -> "ip-192.168.24.12_start_0 controller-2" [ style = bold]
++"ip-192.168.24.12_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume-docker-0_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume-docker-0_start_0 controller-2" -> "openstack-cinder-volume-docker-0_monitor_60000 controller-2" [ style = bold]
++"openstack-cinder-volume-docker-0_start_0 controller-2" -> "openstack-cinder-volume_running_0" [ style = bold]
++"openstack-cinder-volume-docker-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold]
++"openstack-cinder-volume-docker-0_stop_0 controller-0" -> "do_shutdown controller-0" [ style = bold]
++"openstack-cinder-volume-docker-0_stop_0 controller-0" -> "openstack-cinder-volume-docker-0_start_0 controller-2" [ style = bold]
++"openstack-cinder-volume-docker-0_stop_0 controller-0" -> "openstack-cinder-volume_stopped_0" [ style = bold]
++"openstack-cinder-volume-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume_running_0" [ style=bold color="green" fontcolor="orange"]
++"openstack-cinder-volume_start_0" -> "openstack-cinder-volume-docker-0_start_0 controller-2" [ style = bold]
++"openstack-cinder-volume_start_0" [ style=bold color="green" fontcolor="orange"]
++"openstack-cinder-volume_stop_0" -> "openstack-cinder-volume-docker-0_stop_0 controller-0" [ style = bold]
++"openstack-cinder-volume_stop_0" [ style=bold color="green" fontcolor="orange"]
++"openstack-cinder-volume_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-0_monitor_60000 controller-1" [ style=dashed color="red" fontcolor="black"]
++"rabbitmq-bundle-0_start_0 controller-1" -> "rabbitmq-bundle-0_monitor_60000 controller-1" [ style = dashed]
++"rabbitmq-bundle-0_start_0 controller-1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed]
++"rabbitmq-bundle-0_start_0 controller-1" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed]
++"rabbitmq-bundle-0_start_0 controller-1" [ style=dashed color="red" fontcolor="black"]
++"rabbitmq-bundle-0_stop_0 controller-0" -> "all_stopped" [ style = bold]
++"rabbitmq-bundle-0_stop_0 controller-0" -> "do_shutdown controller-0" [ style = bold]
++"rabbitmq-bundle-0_stop_0 controller-0" -> "rabbitmq-bundle-0_start_0 controller-1" [ style = dashed]
++"rabbitmq-bundle-0_stop_0 controller-0" -> "rabbitmq-bundle-docker-0_stop_0 controller-0" [ style = bold]
++"rabbitmq-bundle-0_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle-1_monitor_60000 controller-1" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle-2_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = bold]
++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold]
++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle-clone_pre_notify_start_0" [ style = bold]
++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold]
++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold]
++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold]
++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold]
++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold]
++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold]
++"rabbitmq-bundle-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold]
++"rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stopped_0 rabbitmq-bundle-1" [ style = bold]
++"rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stopped_0 rabbitmq-bundle-2" [ style = bold]
++"rabbitmq-bundle-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold]
++"rabbitmq-bundle-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold]
++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" [ style = bold]
++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" [ style = bold]
++"rabbitmq-bundle-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold]
++"rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed]
++"rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold]
++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold]
++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold]
++"rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold]
++"rabbitmq-bundle-docker-0_stop_0 controller-0" -> "do_shutdown controller-0" [ style = bold]
++"rabbitmq-bundle-docker-0_stop_0 controller-0" -> "rabbitmq-bundle_stopped_0" [ style = bold]
++"rabbitmq-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold]
++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-0_stop_0 controller-0" [ style = bold]
++"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"rabbitmq_post_notify_stopped_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold]
++"rabbitmq_post_notify_stopped_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"]
++"rabbitmq_post_notify_stopped_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold]
++"rabbitmq_post_notify_stopped_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"]
++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold]
++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"]
++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold]
++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"]
++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold]
++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"]
++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed]
++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed]
++"rabbitmq_start_0 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "all_stopped" [ style = bold]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-0_stop_0 controller-0" [ style = bold]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed]
++"rabbitmq_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"]
++}
+diff --git a/pengine/test10/route-remote-notify.exp b/pengine/test10/route-remote-notify.exp
+new file mode 100644
+index 0000000..339555e
+--- /dev/null
++++ b/pengine/test10/route-remote-notify.exp
+@@ -0,0 +1,612 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
++ <synapse id="0">
++ <action_set>
++ <rsc_op id="94" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="controller-0">
++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="rabbitmq-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="53" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="1">
++ <action_set>
++ <rsc_op id="39" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="controller-0">
++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="rabbitmq-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_timeout="200000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="37" operation="stop" operation_key="rabbitmq-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="51" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="2" priority="1000000">
++ <action_set>
++ <rsc_op id="96" operation="notify" operation_key="rabbitmq_post_notify_stop_0" internal_operation_key="rabbitmq:1_post_notify_stop_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="controller-1">
++ <primitive id="rabbitmq" long-id="rabbitmq:1" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="rabbitmq-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="rabbitmq-bundle-1" CRM_meta_on_node_uuid="rabbitmq-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="55" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="3">
++ <action_set>
++ <rsc_op id="95" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:1_pre_notify_stop_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="controller-1">
++ <primitive id="rabbitmq" long-id="rabbitmq:1" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="rabbitmq-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="rabbitmq-bundle-1" CRM_meta_on_node_uuid="rabbitmq-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="53" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="4" priority="1000000">
++ <action_set>
++ <rsc_op id="98" operation="notify" operation_key="rabbitmq_post_notify_stop_0" internal_operation_key="rabbitmq:2_post_notify_stop_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="controller-2">
++ <primitive id="rabbitmq" long-id="rabbitmq:2" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="rabbitmq-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="rabbitmq-bundle-2" CRM_meta_on_node_uuid="rabbitmq-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="55" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="5">
++ <action_set>
++ <rsc_op id="97" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:2_pre_notify_stop_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="controller-2">
++ <primitive id="rabbitmq" long-id="rabbitmq:2" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="rabbitmq:0 rabbitmq:1 rabbitmq:2" CRM_meta_notify_active_uname="rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2" CRM_meta_notify_available_uname="rabbitmq-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="rabbitmq:0" CRM_meta_notify_stop_uname="rabbitmq-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="rabbitmq-bundle-2" CRM_meta_on_node_uuid="rabbitmq-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="53" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="6" priority="1000000">
++ <action_set>
++ <pseudo_event id="56" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-post_notify_stopped_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="55" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_stopped_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="96" operation="notify" operation_key="rabbitmq_post_notify_stop_0" internal_operation_key="rabbitmq:1_post_notify_stop_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="controller-1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="98" operation="notify" operation_key="rabbitmq_post_notify_stop_0" internal_operation_key="rabbitmq:2_post_notify_stop_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="controller-2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7" priority="1000000">
++ <action_set>
++ <pseudo_event id="55" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_stopped_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="52" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="54" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="8">
++ <action_set>
++ <pseudo_event id="54" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_stop_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="53" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="94" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:0_pre_notify_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="controller-0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="95" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:1_pre_notify_stop_0" on_node="rabbitmq-bundle-1" on_node_uuid="rabbitmq-bundle-1" router_node="controller-1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="97" operation="notify" operation_key="rabbitmq_pre_notify_stop_0" internal_operation_key="rabbitmq:2_pre_notify_stop_0" on_node="rabbitmq-bundle-2" on_node_uuid="rabbitmq-bundle-2" router_node="controller-2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="9">
++ <action_set>
++ <pseudo_event id="53" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_stop_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="10" priority="1000000">
++ <action_set>
++ <pseudo_event id="52" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="39" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="controller-0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="51" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="11">
++ <action_set>
++ <pseudo_event id="51" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="37" operation="stop" operation_key="rabbitmq-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="54" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="12" priority="1000000">
++ <action_set>
++ <pseudo_event id="50" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-post_notify_running_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="49" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="13" priority="1000000">
++ <action_set>
++ <pseudo_event id="49" operation="notify" operation_key="rabbitmq-bundle-clone_post_notify_running_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="46" operation="running" operation_key="rabbitmq-bundle-clone_running_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="48" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="14">
++ <action_set>
++ <pseudo_event id="48" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_start_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="47" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="15">
++ <action_set>
++ <pseudo_event id="47" operation="notify" operation_key="rabbitmq-bundle-clone_pre_notify_start_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="56" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-post_notify_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="16" priority="1000000">
++ <action_set>
++ <pseudo_event id="46" operation="running" operation_key="rabbitmq-bundle-clone_running_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="45" operation="start" operation_key="rabbitmq-bundle-clone_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="17">
++ <action_set>
++ <pseudo_event id="45" operation="start" operation_key="rabbitmq-bundle-clone_start_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="48" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-pre_notify_start_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="52" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="18">
++ <action_set>
++ <rsc_op id="21" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="controller-0" on_node_uuid="1">
++ <primitive id="rabbitmq-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/rhosp13/openstack-rabbitmq:pcmklatest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/rabbitmq-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3122 -v /var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/rabbitmq/:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/rabbitmq:/var/lib/rabbitmq:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /var/log/containers/rabbitmq:/var/log/rabbitmq:rw -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/rabbitmq-bundle-0:/var/log --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="22" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="37" operation="stop" operation_key="rabbitmq-bundle_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="19">
++ <action_set>
++ <rsc_op id="22" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="controller-0" on_node_uuid="1">
++ <primitive id="rabbitmq-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="rabbitmq-bundle-docker-0" CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="controller-0" port="3122"/>
++ <downed>
++ <node id="rabbitmq-bundle-0"/>
++ </downed>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="39" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="controller-0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="20">
++ <action_set>
++ <rsc_op id="29" operation="monitor" operation_key="rabbitmq-bundle-1_monitor_60000" on_node="controller-1" on_node_uuid="2">
++ <primitive id="rabbitmq-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="rabbitmq-bundle-docker-1" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="controller-1" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="controller-1" port="3122"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="21">
++ <action_set>
++ <rsc_op id="13" operation="cancel" operation_key="rabbitmq-bundle-1_monitor_30000" on_node="controller-1" on_node_uuid="2">
++ <primitive id="rabbitmq-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_call_id="5" CRM_meta_container="rabbitmq-bundle-docker-1" CRM_meta_interval="30000" CRM_meta_on_node="controller-1" CRM_meta_on_node_uuid="2" CRM_meta_operation="monitor" CRM_meta_timeout="20000" addr="controller-1" port="3122"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="22">
++ <action_set>
++ <rsc_op id="34" operation="monitor" operation_key="rabbitmq-bundle-2_monitor_60000" on_node="controller-2" on_node_uuid="3">
++ <primitive id="rabbitmq-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="rabbitmq-bundle-docker-2" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" addr="controller-2" port="3122"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="23">
++ <action_set>
++ <rsc_op id="8" operation="cancel" operation_key="rabbitmq-bundle-2_monitor_30000" on_node="controller-2" on_node_uuid="3">
++ <primitive id="rabbitmq-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_call_id="5" CRM_meta_container="rabbitmq-bundle-docker-2" CRM_meta_interval="30000" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_operation="monitor" CRM_meta_timeout="20000" addr="controller-2" port="3122"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="24">
++ <action_set>
++ <rsc_op id="59" operation="monitor" operation_key="ip-192.168.24.12_monitor_10000" on_node="controller-2" on_node_uuid="3">
++ <primitive id="ip-192.168.24.12" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.24.12"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="58" operation="start" operation_key="ip-192.168.24.12_start_0" on_node="controller-2" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="25">
++ <action_set>
++ <rsc_op id="58" operation="start" operation_key="ip-192.168.24.12_start_0" on_node="controller-2" on_node_uuid="3">
++ <primitive id="ip-192.168.24.12" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="start" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.24.12"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="57" operation="stop" operation_key="ip-192.168.24.12_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="26">
++ <action_set>
++ <rsc_op id="57" operation="stop" operation_key="ip-192.168.24.12_stop_0" on_node="controller-0" on_node_uuid="1">
++ <primitive id="ip-192.168.24.12" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.24.12"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="79" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="27">
++ <action_set>
++ <rsc_op id="66" operation="monitor" operation_key="ip-172.17.1.11_monitor_10000" on_node="controller-1" on_node_uuid="2">
++ <primitive id="ip-172.17.1.11" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="controller-1" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" cidr_netmask="32" ip="172.17.1.11"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="65" operation="start" operation_key="ip-172.17.1.11_start_0" on_node="controller-1" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="28">
++ <action_set>
++ <rsc_op id="65" operation="start" operation_key="ip-172.17.1.11_start_0" on_node="controller-1" on_node_uuid="2">
++ <primitive id="ip-172.17.1.11" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="start" CRM_meta_on_node="controller-1" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" cidr_netmask="32" ip="172.17.1.11"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="64" operation="stop" operation_key="ip-172.17.1.11_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="29">
++ <action_set>
++ <rsc_op id="64" operation="stop" operation_key="ip-172.17.1.11_stop_0" on_node="controller-0" on_node_uuid="1">
++ <primitive id="ip-172.17.1.11" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="172.17.1.11"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="79" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="30">
++ <action_set>
++ <rsc_op id="71" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="controller-0" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/rhosp13/openstack-haproxy:pcmklatest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/haproxy/:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/haproxy:/var/lib/haproxy:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pki/tls/private/overcloud_endpoint.pem:/var/lib/kolla/config_files/src-tls/etc/pki/tls/private/overcloud_endpoint.pem:ro --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="78" operation="stop" operation_key="haproxy-bundle_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="31">
++ <action_set>
++ <rsc_op id="82" operation="monitor" operation_key="openstack-cinder-volume-docker-0_monitor_60000" on_node="controller-2" on_node_uuid="3">
++ <primitive id="openstack-cinder-volume-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/rhosp13/openstack-cinder-volume:pcmklatest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3121 -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro -v /etc/puppet:/etc/puppet:ro -v /var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/cinder/:/var/lib/kolla/config_files/src:ro -v /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro -v /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro -v /lib/modules:/lib/modules:ro -v /dev/:/dev/:rw -v /run/:/run/:rw -v /sys:/sys:rw -v /var/lib/cinder:/var/lib/cinder:rw -v /var/log/containers/cinder:/var/log/cinder:rw --ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="81" operation="start" operation_key="openstack-cinder-volume-docker-0_start_0" on_node="controller-2" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="32">
++ <action_set>
++ <rsc_op id="81" operation="start" operation_key="openstack-cinder-volume-docker-0_start_0" on_node="controller-2" on_node_uuid="3">
++ <primitive id="openstack-cinder-volume-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/rhosp13/openstack-cinder-volume:pcmklatest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3121 -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro -v /etc/puppet:/etc/puppet:ro -v /var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/cinder/:/var/lib/kolla/config_files/src:ro -v /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro -v /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro -v /lib/modules:/lib/modules:ro -v /dev/:/dev/:rw -v /run/:/run/:rw -v /sys:/sys:rw -v /var/lib/cinder:/var/lib/cinder:rw -v /var/log/containers/cinder:/var/log/cinder:rw --ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="80" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="83" operation="start" operation_key="openstack-cinder-volume_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="33">
++ <action_set>
++ <rsc_op id="80" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="controller-0" on_node_uuid="1">
++ <primitive id="openstack-cinder-volume-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/rhosp13/openstack-cinder-volume:pcmklatest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3121 -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro -v /etc/puppet:/etc/puppet:ro -v /var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/cinder/:/var/lib/kolla/config_files/src:ro -v /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro -v /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro -v /lib/modules:/lib/modules:ro -v /dev/:/dev/:rw -v /run/:/run/:rw -v /sys:/sys:rw -v /var/lib/cinder:/var/lib/cinder:rw -v /var/log/containers/cinder:/var/log/cinder:rw --ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="85" operation="stop" operation_key="openstack-cinder-volume_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="34">
++ <action_set>
++ <crm_event id="88" operation="do_shutdown" operation_key="do_shutdown-controller-0" on_node="controller-0" on_node_uuid="1">
++ <attributes CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_op_no_wait="true" />
++ <downed>
++ <node id="1"/>
++ </downed>
++ </crm_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="21" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="22" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="57" operation="stop" operation_key="ip-192.168.24.12_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="64" operation="stop" operation_key="ip-172.17.1.11_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="71" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="80" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="35" priority="1000000">
++ <action_set>
++ <pseudo_event id="86" operation="stopped" operation_key="openstack-cinder-volume_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="80" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="36">
++ <action_set>
++ <pseudo_event id="85" operation="stop" operation_key="openstack-cinder-volume_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="37" priority="1000000">
++ <action_set>
++ <pseudo_event id="84" operation="running" operation_key="openstack-cinder-volume_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="81" operation="start" operation_key="openstack-cinder-volume-docker-0_start_0" on_node="controller-2" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="38">
++ <action_set>
++ <pseudo_event id="83" operation="start" operation_key="openstack-cinder-volume_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="39" priority="1000000">
++ <action_set>
++ <pseudo_event id="79" operation="stopped" operation_key="haproxy-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="71" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="40">
++ <action_set>
++ <pseudo_event id="78" operation="stop" operation_key="haproxy-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="41" priority="1000000">
++ <action_set>
++ <pseudo_event id="38" operation="stopped" operation_key="rabbitmq-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="21" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="56" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-post_notify_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="42">
++ <action_set>
++ <pseudo_event id="37" operation="stop" operation_key="rabbitmq-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="43" priority="1000000">
++ <action_set>
++ <pseudo_event id="36" operation="running" operation_key="rabbitmq-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="50" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="44">
++ <action_set>
++ <pseudo_event id="20" operation="all_stopped" operation_key="all_stopped">
++ <attributes />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="21" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="22" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="39" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="controller-0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="56" operation="notified" operation_key="rabbitmq-bundle-clone_confirmed-post_notify_stopped_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="57" operation="stop" operation_key="ip-192.168.24.12_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="64" operation="stop" operation_key="ip-172.17.1.11_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="71" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="80" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="controller-0" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++</transition_graph>
+diff --git a/pengine/test10/route-remote-notify.scores b/pengine/test10/route-remote-notify.scores
+new file mode 100644
+index 0000000..ae96db5
+--- /dev/null
++++ b/pengine/test10/route-remote-notify.scores
+@@ -0,0 +1,182 @@
++Allocation scores:
++Using the original execution date of: 2018-10-31 11:51:32Z
++clone_color: rabbitmq-bundle-clone allocation score on controller-0: -INFINITY
++clone_color: rabbitmq-bundle-clone allocation score on controller-1: -INFINITY
++clone_color: rabbitmq-bundle-clone allocation score on controller-2: -INFINITY
++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: 0
++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: 0
++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY
++clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY
++container_color: haproxy-bundle allocation score on controller-0: 0
++container_color: haproxy-bundle allocation score on controller-0: 0
++container_color: haproxy-bundle allocation score on controller-0: 0
++container_color: haproxy-bundle allocation score on controller-0: 0
++container_color: haproxy-bundle allocation score on controller-0: 0
++container_color: haproxy-bundle allocation score on controller-0: 0
++container_color: haproxy-bundle allocation score on controller-0: 0
++container_color: haproxy-bundle allocation score on controller-1: 0
++container_color: haproxy-bundle allocation score on controller-1: 0
++container_color: haproxy-bundle allocation score on controller-1: 0
++container_color: haproxy-bundle allocation score on controller-1: 0
++container_color: haproxy-bundle allocation score on controller-1: 0
++container_color: haproxy-bundle allocation score on controller-1: 0
++container_color: haproxy-bundle allocation score on controller-1: 0
++container_color: haproxy-bundle allocation score on controller-2: 0
++container_color: haproxy-bundle allocation score on controller-2: 0
++container_color: haproxy-bundle allocation score on controller-2: 0
++container_color: haproxy-bundle allocation score on controller-2: 0
++container_color: haproxy-bundle allocation score on controller-2: 0
++container_color: haproxy-bundle allocation score on controller-2: 0
++container_color: haproxy-bundle allocation score on controller-2: 0
++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-0: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-1: 0
++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on controller-2: 0
++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-0: 0
++container_color: haproxy-bundle-docker-1 allocation score on controller-1: INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-1: INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-1: INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-1: INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-1: INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-1: INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-1: INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on controller-2: 0
++container_color: haproxy-bundle-docker-1 allocation score on controller-2: 0
++container_color: haproxy-bundle-docker-1 allocation score on controller-2: 0
++container_color: haproxy-bundle-docker-1 allocation score on controller-2: 0
++container_color: haproxy-bundle-docker-1 allocation score on controller-2: 0
++container_color: haproxy-bundle-docker-1 allocation score on controller-2: 0
++container_color: haproxy-bundle-docker-1 allocation score on controller-2: 0
++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-0: 0
++container_color: haproxy-bundle-docker-2 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-1: 0
++container_color: haproxy-bundle-docker-2 allocation score on controller-2: INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-2: INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-2: INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-2: INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-2: INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-2: INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on controller-2: INFINITY
++container_color: openstack-cinder-volume allocation score on controller-0: 0
++container_color: openstack-cinder-volume allocation score on controller-1: 0
++container_color: openstack-cinder-volume allocation score on controller-2: 0
++container_color: openstack-cinder-volume-docker-0 allocation score on controller-0: INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on controller-1: 0
++container_color: openstack-cinder-volume-docker-0 allocation score on controller-2: 0
++container_color: rabbitmq-bundle allocation score on controller-0: 0
++container_color: rabbitmq-bundle allocation score on controller-1: 0
++container_color: rabbitmq-bundle allocation score on controller-2: 0
++container_color: rabbitmq-bundle-0 allocation score on controller-0: INFINITY
++container_color: rabbitmq-bundle-0 allocation score on controller-1: 0
++container_color: rabbitmq-bundle-0 allocation score on controller-2: 0
++container_color: rabbitmq-bundle-1 allocation score on controller-0: 0
++container_color: rabbitmq-bundle-1 allocation score on controller-1: INFINITY
++container_color: rabbitmq-bundle-1 allocation score on controller-2: 0
++container_color: rabbitmq-bundle-2 allocation score on controller-0: 0
++container_color: rabbitmq-bundle-2 allocation score on controller-1: 0
++container_color: rabbitmq-bundle-2 allocation score on controller-2: INFINITY
++container_color: rabbitmq-bundle-clone allocation score on controller-0: 0
++container_color: rabbitmq-bundle-clone allocation score on controller-1: 0
++container_color: rabbitmq-bundle-clone allocation score on controller-2: 0
++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: -INFINITY
++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: -INFINITY
++container_color: rabbitmq-bundle-docker-0 allocation score on controller-0: INFINITY
++container_color: rabbitmq-bundle-docker-0 allocation score on controller-1: 0
++container_color: rabbitmq-bundle-docker-0 allocation score on controller-2: 0
++container_color: rabbitmq-bundle-docker-1 allocation score on controller-0: 0
++container_color: rabbitmq-bundle-docker-1 allocation score on controller-1: INFINITY
++container_color: rabbitmq-bundle-docker-1 allocation score on controller-2: 0
++container_color: rabbitmq-bundle-docker-2 allocation score on controller-0: 0
++container_color: rabbitmq-bundle-docker-2 allocation score on controller-1: 0
++container_color: rabbitmq-bundle-docker-2 allocation score on controller-2: INFINITY
++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY
++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on controller-1: INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on controller-2: 0
++native_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on controller-1: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on controller-2: INFINITY
++native_color: ip-10.0.0.101 allocation score on controller-0: -INFINITY
++native_color: ip-10.0.0.101 allocation score on controller-1: INFINITY
++native_color: ip-10.0.0.101 allocation score on controller-2: 0
++native_color: ip-172.17.1.11 allocation score on controller-0: -INFINITY
++native_color: ip-172.17.1.11 allocation score on controller-1: 0
++native_color: ip-172.17.1.11 allocation score on controller-2: 0
++native_color: ip-172.17.1.20 allocation score on controller-0: -INFINITY
++native_color: ip-172.17.1.20 allocation score on controller-1: 0
++native_color: ip-172.17.1.20 allocation score on controller-2: INFINITY
++native_color: ip-172.17.3.16 allocation score on controller-0: -INFINITY
++native_color: ip-172.17.3.16 allocation score on controller-1: INFINITY
++native_color: ip-172.17.3.16 allocation score on controller-2: 0
++native_color: ip-172.17.4.15 allocation score on controller-0: -INFINITY
++native_color: ip-172.17.4.15 allocation score on controller-1: 0
++native_color: ip-172.17.4.15 allocation score on controller-2: INFINITY
++native_color: ip-192.168.24.12 allocation score on controller-0: -INFINITY
++native_color: ip-192.168.24.12 allocation score on controller-1: 0
++native_color: ip-192.168.24.12 allocation score on controller-2: 0
++native_color: openstack-cinder-volume-docker-0 allocation score on controller-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on controller-1: 0
++native_color: openstack-cinder-volume-docker-0 allocation score on controller-2: 0
++native_color: rabbitmq-bundle-0 allocation score on controller-0: INFINITY
++native_color: rabbitmq-bundle-0 allocation score on controller-1: 0
++native_color: rabbitmq-bundle-0 allocation score on controller-2: 0
++native_color: rabbitmq-bundle-1 allocation score on controller-0: 0
++native_color: rabbitmq-bundle-1 allocation score on controller-1: INFINITY
++native_color: rabbitmq-bundle-1 allocation score on controller-2: 0
++native_color: rabbitmq-bundle-2 allocation score on controller-0: 0
++native_color: rabbitmq-bundle-2 allocation score on controller-1: 0
++native_color: rabbitmq-bundle-2 allocation score on controller-2: INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on controller-0: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on controller-1: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on controller-2: -INFINITY
++native_color: rabbitmq-bundle-docker-1 allocation score on controller-0: -INFINITY
++native_color: rabbitmq-bundle-docker-1 allocation score on controller-1: INFINITY
++native_color: rabbitmq-bundle-docker-1 allocation score on controller-2: 0
++native_color: rabbitmq-bundle-docker-2 allocation score on controller-0: -INFINITY
++native_color: rabbitmq-bundle-docker-2 allocation score on controller-1: -INFINITY
++native_color: rabbitmq-bundle-docker-2 allocation score on controller-2: INFINITY
++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++native_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY
++native_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY
+diff --git a/pengine/test10/route-remote-notify.summary b/pengine/test10/route-remote-notify.summary
+new file mode 100644
+index 0000000..79b676c
+--- /dev/null
++++ b/pengine/test10/route-remote-notify.summary
+@@ -0,0 +1,102 @@
++Using the original execution date of: 2018-10-31 11:51:32Z
++
++Current cluster status:
++Online: [ controller-0 controller-1 controller-2 ]
++Containers: [ rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 ]
++
++ Docker container set: rabbitmq-bundle [192.168.24.1:8787/rhosp13/openstack-rabbitmq:pcmklatest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started controller-0
++ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started controller-1
++ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started controller-2
++ ip-192.168.24.12 (ocf::heartbeat:IPaddr2): Started controller-0
++ ip-10.0.0.101 (ocf::heartbeat:IPaddr2): Started controller-1
++ ip-172.17.1.20 (ocf::heartbeat:IPaddr2): Started controller-2
++ ip-172.17.1.11 (ocf::heartbeat:IPaddr2): Started controller-0
++ ip-172.17.3.16 (ocf::heartbeat:IPaddr2): Started controller-1
++ ip-172.17.4.15 (ocf::heartbeat:IPaddr2): Started controller-2
++ Docker container set: haproxy-bundle [192.168.24.1:8787/rhosp13/openstack-haproxy:pcmklatest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started controller-0
++ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started controller-1
++ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started controller-2
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/rhosp13/openstack-cinder-volume:pcmklatest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started controller-0
++
++Transition Summary:
++ * Shutdown controller-0
++ * Stop rabbitmq-bundle-docker-0 ( controller-0 ) due to node availability
++ * Stop rabbitmq-bundle-0 ( controller-0 ) due to unrunnable rabbitmq-bundle-docker-0 start
++ * Stop rabbitmq:0 ( rabbitmq-bundle-0 ) due to unrunnable rabbitmq-bundle-docker-0 start
++ * Move ip-192.168.24.12 ( controller-0 -> controller-2 )
++ * Move ip-172.17.1.11 ( controller-0 -> controller-1 )
++ * Stop haproxy-bundle-docker-0 ( controller-0 ) due to node availability
++ * Move openstack-cinder-volume-docker-0 ( controller-0 -> controller-2 )
++
++Executing cluster transition:
++ * Pseudo action: rabbitmq-bundle-clone_pre_notify_stop_0
++ * Resource action: rabbitmq-bundle-1 monitor=60000 on controller-1
++ * Resource action: rabbitmq-bundle-1 cancel=30000 on controller-1
++ * Resource action: rabbitmq-bundle-2 monitor=60000 on controller-2
++ * Resource action: rabbitmq-bundle-2 cancel=30000 on controller-2
++ * Pseudo action: openstack-cinder-volume_stop_0
++ * Pseudo action: openstack-cinder-volume_start_0
++ * Pseudo action: haproxy-bundle_stop_0
++ * Pseudo action: rabbitmq-bundle_stop_0
++ * Resource action: rabbitmq notify on rabbitmq-bundle-0
++ * Resource action: rabbitmq notify on rabbitmq-bundle-1
++ * Resource action: rabbitmq notify on rabbitmq-bundle-2
++ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_stop_0
++ * Pseudo action: rabbitmq-bundle-clone_stop_0
++ * Resource action: haproxy-bundle-docker-0 stop on controller-0
++ * Resource action: openstack-cinder-volume-docker-0 stop on controller-0
++ * Pseudo action: openstack-cinder-volume_stopped_0
++ * Pseudo action: haproxy-bundle_stopped_0
++ * Resource action: rabbitmq stop on rabbitmq-bundle-0
++ * Pseudo action: rabbitmq-bundle-clone_stopped_0
++ * Resource action: rabbitmq-bundle-0 stop on controller-0
++ * Resource action: ip-192.168.24.12 stop on controller-0
++ * Resource action: ip-172.17.1.11 stop on controller-0
++ * Resource action: openstack-cinder-volume-docker-0 start on controller-2
++ * Pseudo action: openstack-cinder-volume_running_0
++ * Pseudo action: rabbitmq-bundle-clone_post_notify_stopped_0
++ * Resource action: rabbitmq-bundle-docker-0 stop on controller-0
++ * Resource action: ip-192.168.24.12 start on controller-2
++ * Resource action: ip-172.17.1.11 start on controller-1
++ * Resource action: openstack-cinder-volume-docker-0 monitor=60000 on controller-2
++ * Cluster action: do_shutdown on controller-0
++ * Resource action: rabbitmq notify on rabbitmq-bundle-1
++ * Resource action: rabbitmq notify on rabbitmq-bundle-2
++ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_stopped_0
++ * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0
++ * Resource action: ip-192.168.24.12 monitor=10000 on controller-2
++ * Resource action: ip-172.17.1.11 monitor=10000 on controller-1
++ * Pseudo action: rabbitmq-bundle_stopped_0
++ * Pseudo action: all_stopped
++ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0
++ * Pseudo action: rabbitmq-bundle-clone_start_0
++ * Pseudo action: rabbitmq-bundle-clone_running_0
++ * Pseudo action: rabbitmq-bundle-clone_post_notify_running_0
++ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_running_0
++ * Pseudo action: rabbitmq-bundle_running_0
++Using the original execution date of: 2018-10-31 11:51:32Z
++
++Revised cluster status:
++Online: [ controller-0 controller-1 controller-2 ]
++Containers: [ rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 ]
++
++ Docker container set: rabbitmq-bundle [192.168.24.1:8787/rhosp13/openstack-rabbitmq:pcmklatest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped
++ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started controller-1
++ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started controller-2
++ ip-192.168.24.12 (ocf::heartbeat:IPaddr2): Started controller-2
++ ip-10.0.0.101 (ocf::heartbeat:IPaddr2): Started controller-1
++ ip-172.17.1.20 (ocf::heartbeat:IPaddr2): Started controller-2
++ ip-172.17.1.11 (ocf::heartbeat:IPaddr2): Started controller-1
++ ip-172.17.3.16 (ocf::heartbeat:IPaddr2): Started controller-1
++ ip-172.17.4.15 (ocf::heartbeat:IPaddr2): Started controller-2
++ Docker container set: haproxy-bundle [192.168.24.1:8787/rhosp13/openstack-haproxy:pcmklatest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped
++ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started controller-1
++ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started controller-2
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/rhosp13/openstack-cinder-volume:pcmklatest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started controller-2
++
+diff --git a/pengine/test10/route-remote-notify.xml b/pengine/test10/route-remote-notify.xml
+new file mode 100644
+index 0000000..0beba46
+--- /dev/null
++++ b/pengine/test10/route-remote-notify.xml
+@@ -0,0 +1,487 @@
++<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.10" epoch="74" num_updates="5" admin_epoch="0" cib-last-written="Wed Oct 31 11:26:39 2018" update-origin="controller-0" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="3" execution-date="1540986692">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.19-8.el7-c3c624ea3d"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="controller-0">
++ <instance_attributes id="nodes-1">
++ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-1-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@controller-0"/>
++ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="2" uname="controller-1">
++ <instance_attributes id="nodes-2">
++ <nvpair id="nodes-2-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-2-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@controller-1"/>
++ <nvpair id="nodes-2-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-2-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="3" uname="controller-2">
++ <instance_attributes id="nodes-3">
++ <nvpair id="nodes-3-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-3-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@controller-2"/>
++ <nvpair id="nodes-3-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-3-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ </nodes>
++ <resources>
++ <bundle id="rabbitmq-bundle">
++ <docker image="192.168.24.1:8787/rhosp13/openstack-rabbitmq:pcmklatest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="3" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3122"/>
++ <storage>
++ <storage-mapping id="rabbitmq-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/rabbitmq.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="rabbitmq-cfg-data" options="ro" source-dir="/var/lib/config-data/puppet-generated/rabbitmq/" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="rabbitmq-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="rabbitmq-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="rabbitmq-lib" options="rw" source-dir="/var/lib/rabbitmq" target-dir="/var/lib/rabbitmq"/>
++ <storage-mapping id="rabbitmq-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="rabbitmq-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="rabbitmq-log" options="rw" source-dir="/var/log/containers/rabbitmq" target-dir="/var/log/rabbitmq"/>
++ <storage-mapping id="rabbitmq-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="rabbitmq" provider="heartbeat" type="rabbitmq-cluster">
++ <instance_attributes id="rabbitmq-instance_attributes">
++ <nvpair id="rabbitmq-instance_attributes-set_policy" name="set_policy" value="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </instance_attributes>
++ <meta_attributes id="rabbitmq-meta_attributes">
++ <nvpair id="rabbitmq-meta_attributes-container-attribute-target" name="container-attribute-target" value="host"/>
++ <nvpair id="rabbitmq-meta_attributes-notify" name="notify" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="rabbitmq-monitor-interval-10s" interval="10s" name="monitor" timeout="40s"/>
++ <op id="rabbitmq-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="rabbitmq-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <primitive class="ocf" id="ip-192.168.24.12" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.24.12-instance_attributes">
++ <nvpair id="ip-192.168.24.12-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.24.12-instance_attributes-ip" name="ip" value="192.168.24.12"/>
++ </instance_attributes>
++ <meta_attributes id="ip-192.168.24.12-meta_attributes">
++ <nvpair id="ip-192.168.24.12-meta_attributes-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ <operations>
++ <op id="ip-192.168.24.12-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.24.12-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.24.12-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-10.0.0.101" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-10.0.0.101-instance_attributes">
++ <nvpair id="ip-10.0.0.101-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-10.0.0.101-instance_attributes-ip" name="ip" value="10.0.0.101"/>
++ </instance_attributes>
++ <meta_attributes id="ip-10.0.0.101-meta_attributes">
++ <nvpair id="ip-10.0.0.101-meta_attributes-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ <operations>
++ <op id="ip-10.0.0.101-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-10.0.0.101-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-10.0.0.101-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-172.17.1.20" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-172.17.1.20-instance_attributes">
++ <nvpair id="ip-172.17.1.20-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-172.17.1.20-instance_attributes-ip" name="ip" value="172.17.1.20"/>
++ </instance_attributes>
++ <meta_attributes id="ip-172.17.1.20-meta_attributes">
++ <nvpair id="ip-172.17.1.20-meta_attributes-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ <operations>
++ <op id="ip-172.17.1.20-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-172.17.1.20-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-172.17.1.20-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-172.17.1.11" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-172.17.1.11-instance_attributes">
++ <nvpair id="ip-172.17.1.11-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-172.17.1.11-instance_attributes-ip" name="ip" value="172.17.1.11"/>
++ </instance_attributes>
++ <meta_attributes id="ip-172.17.1.11-meta_attributes">
++ <nvpair id="ip-172.17.1.11-meta_attributes-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ <operations>
++ <op id="ip-172.17.1.11-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-172.17.1.11-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-172.17.1.11-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-172.17.3.16" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-172.17.3.16-instance_attributes">
++ <nvpair id="ip-172.17.3.16-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-172.17.3.16-instance_attributes-ip" name="ip" value="172.17.3.16"/>
++ </instance_attributes>
++ <meta_attributes id="ip-172.17.3.16-meta_attributes">
++ <nvpair id="ip-172.17.3.16-meta_attributes-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ <operations>
++ <op id="ip-172.17.3.16-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-172.17.3.16-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-172.17.3.16-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-172.17.4.15" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-172.17.4.15-instance_attributes">
++ <nvpair id="ip-172.17.4.15-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-172.17.4.15-instance_attributes-ip" name="ip" value="172.17.4.15"/>
++ </instance_attributes>
++ <meta_attributes id="ip-172.17.4.15-meta_attributes">
++ <nvpair id="ip-172.17.4.15-meta_attributes-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ <operations>
++ <op id="ip-172.17.4.15-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-172.17.4.15-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-172.17.4.15-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <bundle id="haproxy-bundle">
++ <docker image="192.168.24.1:8787/rhosp13/openstack-haproxy:pcmklatest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="3" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <storage>
++ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/puppet-generated/haproxy/" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="haproxy-var-lib" options="rw" source-dir="/var/lib/haproxy" target-dir="/var/lib/haproxy"/>
++ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ <storage-mapping id="haproxy-cert" options="ro" source-dir="/etc/pki/tls/private/overcloud_endpoint.pem" target-dir="/var/lib/kolla/config_files/src-tls/etc/pki/tls/private/overcloud_endpoint.pem"/>
++ </storage>
++ </bundle>
++ <bundle id="openstack-cinder-volume">
++ <docker image="192.168.24.1:8787/rhosp13/openstack-cinder-volume:pcmklatest" network="host" options="--ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <storage>
++ <storage-mapping id="cinder-volume-etc-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="cinder-volume-etc-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="cinder-volume-etc-pki-ca-trust-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="cinder-volume-etc-pki-ca-trust-source-anchors" options="ro" source-dir="/etc/pki/ca-trust/source/anchors" target-dir="/etc/pki/ca-trust/source/anchors"/>
++ <storage-mapping id="cinder-volume-etc-pki-tls-certs-ca-bundle.crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="cinder-volume-etc-pki-tls-certs-ca-bundle.trust.crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="cinder-volume-etc-pki-tls-cert.pem" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="cinder-volume-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ <storage-mapping id="cinder-volume-etc-ssh-ssh_known_hosts" options="ro" source-dir="/etc/ssh/ssh_known_hosts" target-dir="/etc/ssh/ssh_known_hosts"/>
++ <storage-mapping id="cinder-volume-etc-puppet" options="ro" source-dir="/etc/puppet" target-dir="/etc/puppet"/>
++ <storage-mapping id="cinder-volume-var-lib-kolla-config_files-cinder_volume.json" options="ro" source-dir="/var/lib/kolla/config_files/cinder_volume.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="cinder-volume-var-lib-config-data-puppet-generated-cinder-" options="ro" source-dir="/var/lib/config-data/puppet-generated/cinder/" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="cinder-volume-etc-iscsi" options="ro" source-dir="/etc/iscsi" target-dir="/var/lib/kolla/config_files/src-iscsid"/>
++ <storage-mapping id="cinder-volume-etc-ceph" options="ro" source-dir="/etc/ceph" target-dir="/var/lib/kolla/config_files/src-ceph"/>
++ <storage-mapping id="cinder-volume-lib-modules" options="ro" source-dir="/lib/modules" target-dir="/lib/modules"/>
++ <storage-mapping id="cinder-volume-dev-" options="rw" source-dir="/dev/" target-dir="/dev/"/>
++ <storage-mapping id="cinder-volume-run-" options="rw" source-dir="/run/" target-dir="/run/"/>
++ <storage-mapping id="cinder-volume-sys" options="rw" source-dir="/sys" target-dir="/sys"/>
++ <storage-mapping id="cinder-volume-var-lib-cinder" options="rw" source-dir="/var/lib/cinder" target-dir="/var/lib/cinder"/>
++ <storage-mapping id="cinder-volume-var-log-containers-cinder" options="rw" source-dir="/var/log/containers/cinder" target-dir="/var/log/cinder"/>
++ </storage>
++ </bundle>
++ </resources>
++ <constraints>
++ <rsc_location id="location-rabbitmq-bundle" resource-discovery="exclusive" rsc="rabbitmq-bundle">
++ <rule id="location-rabbitmq-bundle-rule" score="0">
++ <expression attribute="rabbitmq-role" id="location-rabbitmq-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.24.12" resource-discovery="exclusive" rsc="ip-192.168.24.12">
++ <rule id="location-ip-192.168.24.12-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.24.12-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-10.0.0.101" resource-discovery="exclusive" rsc="ip-10.0.0.101">
++ <rule id="location-ip-10.0.0.101-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-10.0.0.101-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-172.17.1.20" resource-discovery="exclusive" rsc="ip-172.17.1.20">
++ <rule id="location-ip-172.17.1.20-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-172.17.1.20-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-172.17.1.11" resource-discovery="exclusive" rsc="ip-172.17.1.11">
++ <rule id="location-ip-172.17.1.11-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-172.17.1.11-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-172.17.3.16" resource-discovery="exclusive" rsc="ip-172.17.3.16">
++ <rule id="location-ip-172.17.3.16-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-172.17.3.16-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-172.17.4.15" resource-discovery="exclusive" rsc="ip-172.17.4.15">
++ <rule id="location-ip-172.17.4.15-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-172.17.4.15-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-haproxy-bundle" resource-discovery="exclusive" rsc="haproxy-bundle">
++ <rule id="location-haproxy-bundle-rule" score="0">
++ <expression attribute="haproxy-role" id="location-haproxy-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_order first="ip-192.168.24.12" first-action="start" id="order-ip-192.168.24.12-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.24.12-haproxy-bundle-INFINITY" rsc="ip-192.168.24.12" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-10.0.0.101" first-action="start" id="order-ip-10.0.0.101-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-10.0.0.101-haproxy-bundle-INFINITY" rsc="ip-10.0.0.101" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-172.17.1.20" first-action="start" id="order-ip-172.17.1.20-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-172.17.1.20-haproxy-bundle-INFINITY" rsc="ip-172.17.1.20" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-172.17.1.11" first-action="start" id="order-ip-172.17.1.11-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-172.17.1.11-haproxy-bundle-INFINITY" rsc="ip-172.17.1.11" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-172.17.3.16" first-action="start" id="order-ip-172.17.3.16-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-172.17.3.16-haproxy-bundle-INFINITY" rsc="ip-172.17.3.16" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-172.17.4.15" first-action="start" id="order-ip-172.17.4.15-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-172.17.4.15-haproxy-bundle-INFINITY" rsc="ip-172.17.4.15" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_location id="location-openstack-cinder-volume" resource-discovery="exclusive" rsc="openstack-cinder-volume">
++ <rule id="location-openstack-cinder-volume-rule" score="0">
++ <expression attribute="cinder-volume-role" id="location-openstack-cinder-volume-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ </constraints>
++ <rsc_defaults>
++ <meta_attributes id="rsc_defaults-options">
++ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ </rsc_defaults>
++ </configuration>
++ <status>
++ <node_state id="1" uname="controller-0" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="1">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="5:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;5:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1540983900" last-rc-change="1540983900" exec-time="1083" queue-time="0" op-digest="ae040b82313c323e21c4670707a9780a"/>
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_monitor_60000" operation_key="rabbitmq-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="1:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;1:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="21" rc-code="0" op-status="0" interval="60000" last-rc-change="1540983901" exec-time="337" queue-time="0" op-digest="3a23c9f953086b4add9203bf80f77e0f"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-1_last_0" operation_key="rabbitmq-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="3:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;3:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1540983868" last-rc-change="1540983868" exec-time="196" queue-time="0" op-digest="c9cf2c04cfd190d4d1d069c0891074fb"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-2_last_0" operation_key="rabbitmq-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="4:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;4:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="13" rc-code="7" op-status="0" interval="0" last-run="1540983868" last-rc-change="1540983868" exec-time="195" queue-time="0" op-digest="c9cf2c04cfd190d4d1d069c0891074fb"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-2" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-2">
++ <lrm_rsc_op id="rabbitmq-bundle-2_last_0" operation_key="rabbitmq-bundle-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="10:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;10:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1540983901" last-rc-change="1540983901" exec-time="0" queue-time="0" op-digest="205c791aea96714f7aa1cb111c727e1b" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-1" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-1">
++ <lrm_rsc_op id="rabbitmq-bundle-1_last_0" operation_key="rabbitmq-bundle-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="9:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;9:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1540983901" last-rc-change="1540983901" exec-time="0" queue-time="0" op-digest="f1d68ab267df6867301fcff08041cd2f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0">
++ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="21:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;21:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="4" rc-code="0" op-status="0" interval="0" last-run="1540983901" last-rc-change="1540983901" exec-time="0" queue-time="0" op-digest="8dc3f9844e70d4458a7edcd7091ecf50" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="rabbitmq-bundle-0_monitor_30000" operation_key="rabbitmq-bundle-0_monitor_30000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="12:10:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;12:10:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="5" rc-code="0" op-status="0" interval="30000" last-rc-change="1540983902" exec-time="0" queue-time="0" op-digest="cca9d25863986607e6440270f0640ff5"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.24.12" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.24.12_last_0" operation_key="ip-192.168.24.12_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="157:41:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;157:41:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="69" rc-code="0" op-status="0" interval="0" last-run="1540984260" last-rc-change="1540984260" exec-time="199" queue-time="1" op-digest="b4014942f7386bbb0f624129d6226304"/>
++ <lrm_rsc_op id="ip-192.168.24.12_monitor_10000" operation_key="ip-192.168.24.12_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="158:41:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;158:41:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="70" rc-code="0" op-status="0" interval="10000" last-rc-change="1540984260" exec-time="77" queue-time="0" op-digest="62c2cc737006fd52cbe49b4d8368ef2c"/>
++ </lrm_resource>
++ <lrm_resource id="ip-10.0.0.101" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-10.0.0.101_last_0" operation_key="ip-10.0.0.101_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="32:42:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;32:42:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="74" rc-code="7" op-status="0" interval="0" last-run="1540984269" last-rc-change="1540984269" exec-time="78" queue-time="0" op-digest="cd1adb3084285dafcc7313900769abf2"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.1.20" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.1.20_last_0" operation_key="ip-172.17.1.20_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="33:45:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;33:45:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="78" rc-code="7" op-status="0" interval="0" last-run="1540984284" last-rc-change="1540984284" exec-time="117" queue-time="0" op-digest="1a1c49003092bcbbe8185669d9ca3980"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.1.11" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.1.11_last_0" operation_key="ip-172.17.1.11_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="166:50:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;166:50:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="83" rc-code="0" op-status="0" interval="0" last-run="1540984306" last-rc-change="1540984306" exec-time="165" queue-time="0" op-digest="7c6ed186ff8cdf7693b51ee42b4fc6e1"/>
++ <lrm_rsc_op id="ip-172.17.1.11_monitor_10000" operation_key="ip-172.17.1.11_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="167:50:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;167:50:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="84" rc-code="0" op-status="0" interval="10000" last-rc-change="1540984306" exec-time="59" queue-time="0" op-digest="37c8f827f04bb4d80e7fee9c98c84da2"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.3.16" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.3.16_last_0" operation_key="ip-172.17.3.16_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="35:51:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;35:51:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="88" rc-code="7" op-status="0" interval="0" last-run="1540984315" last-rc-change="1540984315" exec-time="102" queue-time="0" op-digest="95a6216103fdc21c7c2ebc7a129b5b2c"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.4.15" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.4.15_last_0" operation_key="ip-172.17.4.15_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="36:54:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;36:54:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="92" rc-code="7" op-status="0" interval="0" last-run="1540984330" last-rc-change="1540984330" exec-time="103" queue-time="0" op-digest="4f62c49dc50d6bdde262b2bbd44cd9b6"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="175:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;175:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="105" rc-code="0" op-status="0" interval="0" last-run="1540984348" last-rc-change="1540984348" exec-time="913" queue-time="0" op-digest="416854bcd50cb9d353ec34d0d1bec66f"/>
++ <lrm_rsc_op id="haproxy-bundle-docker-0_monitor_60000" operation_key="haproxy-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="176:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;176:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="106" rc-code="0" op-status="0" interval="60000" last-rc-change="1540984349" exec-time="527" queue-time="0" op-digest="ce95c0cdd076935266f1acb8e1f8de41"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-1_last_0" operation_key="haproxy-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="38:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;38:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="100" rc-code="7" op-status="0" interval="0" last-run="1540984342" last-rc-change="1540984342" exec-time="162" queue-time="0" op-digest="416854bcd50cb9d353ec34d0d1bec66f"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-2_last_0" operation_key="haproxy-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="39:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;39:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="104" rc-code="7" op-status="0" interval="0" last-run="1540984342" last-rc-change="1540984342" exec-time="177" queue-time="0" op-digest="416854bcd50cb9d353ec34d0d1bec66f"/>
++ </lrm_resource>
++ <lrm_resource id="openstack-cinder-volume-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_0" operation_key="openstack-cinder-volume-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="188:77:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;188:77:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="111" rc-code="0" op-status="0" interval="0" last-run="1540985199" last-rc-change="1540985199" exec-time="1576" queue-time="0" op-digest="13e8c88fda0d2de2e026824b90a4acc3"/>
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_monitor_60000" operation_key="openstack-cinder-volume-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="189:77:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;189:77:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="112" rc-code="0" op-status="0" interval="60000" last-rc-change="1540985201" exec-time="542" queue-time="0" op-digest="0e6bd23d2a96350c5b4c6b5578024885"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="1">
++ <instance_attributes id="status-1">
++ <nvpair id="status-1-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@controller-0"/>
++ <nvpair id="status-1-shutdown" name="shutdown" value="1540986692"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ <node_state id="3" uname="controller-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="3">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="8:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;8:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1540983868" last-rc-change="1540983868" exec-time="187" queue-time="0" op-digest="c9cf2c04cfd190d4d1d069c0891074fb"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-1_last_0" operation_key="rabbitmq-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="9:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;9:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1540983868" last-rc-change="1540983868" exec-time="106" queue-time="0" op-digest="c9cf2c04cfd190d4d1d069c0891074fb"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-2_last_0" operation_key="rabbitmq-bundle-docker-2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="6:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;6:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1540983900" last-rc-change="1540983900" exec-time="1041" queue-time="0" op-digest="a598abeee324765f1126ada0550bfe50"/>
++ <lrm_rsc_op id="rabbitmq-bundle-docker-2_monitor_60000" operation_key="rabbitmq-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="2:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;2:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="19" rc-code="0" op-status="0" interval="60000" last-rc-change="1540983901" exec-time="293" queue-time="0" op-digest="9b2e69608eaf664ce11b97a6e2a95a62"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-2" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-2">
++ <lrm_rsc_op id="rabbitmq-bundle-2_last_0" operation_key="rabbitmq-bundle-2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="27:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;27:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="4" rc-code="0" op-status="0" interval="0" last-run="1540983901" last-rc-change="1540983901" exec-time="0" queue-time="0" op-digest="205c791aea96714f7aa1cb111c727e1b" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="rabbitmq-bundle-2_monitor_30000" operation_key="rabbitmq-bundle-2_monitor_30000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="22:10:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;22:10:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="5" rc-code="0" op-status="0" interval="30000" last-rc-change="1540983902" exec-time="0" queue-time="0" op-digest="f765304166a359903e5aea671a7275d0"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-1" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-1">
++ <lrm_rsc_op id="rabbitmq-bundle-1_last_0" operation_key="rabbitmq-bundle-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="15:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;15:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1540983901" last-rc-change="1540983901" exec-time="0" queue-time="0" op-digest="f1d68ab267df6867301fcff08041cd2f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0">
++ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="14:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;14:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1540983901" last-rc-change="1540983901" exec-time="0" queue-time="0" op-digest="8dc3f9844e70d4458a7edcd7091ecf50" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.24.12" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.24.12_last_0" operation_key="ip-192.168.24.12_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="33:39:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;33:39:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="68" rc-code="7" op-status="0" interval="0" last-run="1540984254" last-rc-change="1540984254" exec-time="111" queue-time="0" op-digest="b4014942f7386bbb0f624129d6226304"/>
++ </lrm_resource>
++ <lrm_resource id="ip-10.0.0.101" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-10.0.0.101_last_0" operation_key="ip-10.0.0.101_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="34:42:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;34:42:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="72" rc-code="7" op-status="0" interval="0" last-run="1540984269" last-rc-change="1540984269" exec-time="62" queue-time="0" op-digest="cd1adb3084285dafcc7313900769abf2"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.1.20" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.1.20_last_0" operation_key="ip-172.17.1.20_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="163:47:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;163:47:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="77" rc-code="0" op-status="0" interval="0" last-run="1540984291" last-rc-change="1540984291" exec-time="93" queue-time="0" op-digest="1a1c49003092bcbbe8185669d9ca3980"/>
++ <lrm_rsc_op id="ip-172.17.1.20_monitor_10000" operation_key="ip-172.17.1.20_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="164:47:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;164:47:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="78" rc-code="0" op-status="0" interval="10000" last-rc-change="1540984291" exec-time="81" queue-time="0" op-digest="326874e77f47e2c6e5ed34b617b539ea"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.1.11" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.1.11_last_0" operation_key="ip-172.17.1.11_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="36:48:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;36:48:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="82" rc-code="7" op-status="0" interval="0" last-run="1540984299" last-rc-change="1540984299" exec-time="78" queue-time="0" op-digest="7c6ed186ff8cdf7693b51ee42b4fc6e1"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.3.16" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.3.16_last_0" operation_key="ip-172.17.3.16_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="37:51:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;37:51:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="86" rc-code="7" op-status="0" interval="0" last-run="1540984315" last-rc-change="1540984315" exec-time="72" queue-time="0" op-digest="95a6216103fdc21c7c2ebc7a129b5b2c"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.4.15" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.4.15_last_0" operation_key="ip-172.17.4.15_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="172:56:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;172:56:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="91" rc-code="0" op-status="0" interval="0" last-run="1540984336" last-rc-change="1540984336" exec-time="134" queue-time="0" op-digest="4f62c49dc50d6bdde262b2bbd44cd9b6"/>
++ <lrm_rsc_op id="ip-172.17.4.15_monitor_10000" operation_key="ip-172.17.4.15_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="173:56:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;173:56:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="92" rc-code="0" op-status="0" interval="10000" last-rc-change="1540984336" exec-time="69" queue-time="0" op-digest="132812b94329474c80e2c7e421722bfa"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="43:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;43:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="96" rc-code="7" op-status="0" interval="0" last-run="1540984342" last-rc-change="1540984342" exec-time="101" queue-time="0" op-digest="416854bcd50cb9d353ec34d0d1bec66f"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-1_last_0" operation_key="haproxy-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="44:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;44:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="100" rc-code="7" op-status="0" interval="0" last-run="1540984342" last-rc-change="1540984342" exec-time="121" queue-time="0" op-digest="416854bcd50cb9d353ec34d0d1bec66f"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-2_last_0" operation_key="haproxy-bundle-docker-2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="179:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;179:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="105" rc-code="0" op-status="0" interval="0" last-run="1540984348" last-rc-change="1540984348" exec-time="1248" queue-time="1" op-digest="416854bcd50cb9d353ec34d0d1bec66f"/>
++ <lrm_rsc_op id="haproxy-bundle-docker-2_monitor_60000" operation_key="haproxy-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="180:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;180:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="106" rc-code="0" op-status="0" interval="60000" last-rc-change="1540984349" exec-time="430" queue-time="0" op-digest="ce95c0cdd076935266f1acb8e1f8de41"/>
++ </lrm_resource>
++ <lrm_resource id="openstack-cinder-volume-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_0" operation_key="openstack-cinder-volume-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="42:75:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;42:75:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="110" rc-code="7" op-status="0" interval="0" last-run="1540985187" last-rc-change="1540985187" exec-time="114" queue-time="1" op-digest="13e8c88fda0d2de2e026824b90a4acc3"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="3">
++ <instance_attributes id="status-3">
++ <nvpair id="status-3-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@controller-2"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ <node_state id="2" uname="controller-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="2">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="5:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;5:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1540983868" last-rc-change="1540983868" exec-time="178" queue-time="0" op-digest="c9cf2c04cfd190d4d1d069c0891074fb"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-1_last_0" operation_key="rabbitmq-bundle-docker-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;7:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1540983900" last-rc-change="1540983900" exec-time="1136" queue-time="0" op-digest="96dff92057862df79b06a95f8d1a9b2e"/>
++ <lrm_rsc_op id="rabbitmq-bundle-docker-1_monitor_60000" operation_key="rabbitmq-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="3:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;3:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="20" rc-code="0" op-status="0" interval="60000" last-rc-change="1540983901" exec-time="280" queue-time="0" op-digest="391eec1a0e54c3c419b584cbb13f79e8"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-2_last_0" operation_key="rabbitmq-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;7:6:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="13" rc-code="7" op-status="0" interval="0" last-run="1540983868" last-rc-change="1540983868" exec-time="164" queue-time="0" op-digest="c9cf2c04cfd190d4d1d069c0891074fb"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-2" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-2">
++ <lrm_rsc_op id="rabbitmq-bundle-2_last_0" operation_key="rabbitmq-bundle-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="13:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;13:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1540983901" last-rc-change="1540983901" exec-time="0" queue-time="0" op-digest="205c791aea96714f7aa1cb111c727e1b" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-1" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-1">
++ <lrm_rsc_op id="rabbitmq-bundle-1_last_0" operation_key="rabbitmq-bundle-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="24:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;24:9:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="4" rc-code="0" op-status="0" interval="0" last-run="1540983901" last-rc-change="1540983901" exec-time="0" queue-time="0" op-digest="f1d68ab267df6867301fcff08041cd2f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="rabbitmq-bundle-1_monitor_30000" operation_key="rabbitmq-bundle-1_monitor_30000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="17:10:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;17:10:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="5" rc-code="0" op-status="0" interval="30000" last-rc-change="1540983902" exec-time="0" queue-time="0" op-digest="61c3a926b0d3bd428b481dd6e40b1745"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0">
++ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="11:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;11:9:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1540983901" last-rc-change="1540983901" exec-time="0" queue-time="0" op-digest="8dc3f9844e70d4458a7edcd7091ecf50" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.24.12" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.24.12_last_0" operation_key="ip-192.168.24.12_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="32:39:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;32:39:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="68" rc-code="7" op-status="0" interval="0" last-run="1540984254" last-rc-change="1540984254" exec-time="91" queue-time="0" op-digest="b4014942f7386bbb0f624129d6226304"/>
++ </lrm_resource>
++ <lrm_resource id="ip-10.0.0.101" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-10.0.0.101_last_0" operation_key="ip-10.0.0.101_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="160:44:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;160:44:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="73" rc-code="0" op-status="0" interval="0" last-run="1540984275" last-rc-change="1540984275" exec-time="150" queue-time="0" op-digest="cd1adb3084285dafcc7313900769abf2"/>
++ <lrm_rsc_op id="ip-10.0.0.101_monitor_10000" operation_key="ip-10.0.0.101_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="161:44:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;161:44:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="74" rc-code="0" op-status="0" interval="10000" last-rc-change="1540984275" exec-time="83" queue-time="0" op-digest="70a7b88579e4b905ecb297795e1f94a8"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.1.20" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.1.20_last_0" operation_key="ip-172.17.1.20_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="34:45:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;34:45:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="78" rc-code="7" op-status="0" interval="0" last-run="1540984284" last-rc-change="1540984284" exec-time="101" queue-time="0" op-digest="1a1c49003092bcbbe8185669d9ca3980"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.1.11" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.1.11_last_0" operation_key="ip-172.17.1.11_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="35:48:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;35:48:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="82" rc-code="7" op-status="0" interval="0" last-run="1540984299" last-rc-change="1540984299" exec-time="69" queue-time="0" op-digest="7c6ed186ff8cdf7693b51ee42b4fc6e1"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.3.16" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.3.16_last_0" operation_key="ip-172.17.3.16_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="169:53:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;169:53:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="87" rc-code="0" op-status="0" interval="0" last-run="1540984321" last-rc-change="1540984321" exec-time="290" queue-time="0" op-digest="95a6216103fdc21c7c2ebc7a129b5b2c"/>
++ <lrm_rsc_op id="ip-172.17.3.16_monitor_10000" operation_key="ip-172.17.3.16_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="170:53:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;170:53:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="88" rc-code="0" op-status="0" interval="10000" last-rc-change="1540984321" exec-time="113" queue-time="0" op-digest="85339a807a17f5e5caf2167def89d03d"/>
++ </lrm_resource>
++ <lrm_resource id="ip-172.17.4.15" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-172.17.4.15_last_0" operation_key="ip-172.17.4.15_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="37:54:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;37:54:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="92" rc-code="7" op-status="0" interval="0" last-run="1540984330" last-rc-change="1540984330" exec-time="99" queue-time="0" op-digest="4f62c49dc50d6bdde262b2bbd44cd9b6"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="40:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;40:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="96" rc-code="7" op-status="0" interval="0" last-run="1540984342" last-rc-change="1540984342" exec-time="166" queue-time="0" op-digest="416854bcd50cb9d353ec34d0d1bec66f"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-1_last_0" operation_key="haproxy-bundle-docker-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="177:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;177:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="105" rc-code="0" op-status="0" interval="0" last-run="1540984348" last-rc-change="1540984348" exec-time="1077" queue-time="0" op-digest="416854bcd50cb9d353ec34d0d1bec66f"/>
++ <lrm_rsc_op id="haproxy-bundle-docker-1_monitor_60000" operation_key="haproxy-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="178:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;178:59:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="106" rc-code="0" op-status="0" interval="60000" last-rc-change="1540984349" exec-time="692" queue-time="0" op-digest="ce95c0cdd076935266f1acb8e1f8de41"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-2_last_0" operation_key="haproxy-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="42:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;42:57:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="104" rc-code="7" op-status="0" interval="0" last-run="1540984342" last-rc-change="1540984342" exec-time="187" queue-time="0" op-digest="416854bcd50cb9d353ec34d0d1bec66f"/>
++ </lrm_resource>
++ <lrm_resource id="openstack-cinder-volume-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_0" operation_key="openstack-cinder-volume-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="41:75:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:7;41:75:7:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="110" rc-code="7" op-status="0" interval="0" last-run="1540985188" last-rc-change="1540985188" exec-time="145" queue-time="0" op-digest="13e8c88fda0d2de2e026824b90a4acc3"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="2">
++ <instance_attributes id="status-2">
++ <nvpair id="status-2-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@controller-1"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ <node_state remote_node="true" id="rabbitmq-bundle-2" uname="rabbitmq-bundle-2" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <lrm id="rabbitmq-bundle-2">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="30:12:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;30:12:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="15" rc-code="0" op-status="0" interval="0" last-run="1540983948" last-rc-change="1540983948" exec-time="28858" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/>
++ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="32:13:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;32:13:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-2" call-id="54" rc-code="0" op-status="0" interval="10000" last-rc-change="1540983978" exec-time="5645" queue-time="0" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state remote_node="true" id="rabbitmq-bundle-1" uname="rabbitmq-bundle-1" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <lrm id="rabbitmq-bundle-1">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="27:11:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;27:11:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="15" rc-code="0" op-status="0" interval="0" last-run="1540983923" last-rc-change="1540983923" exec-time="23986" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/>
++ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="29:13:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;29:13:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-1" call-id="55" rc-code="0" op-status="0" interval="10000" last-rc-change="1540983978" exec-time="5566" queue-time="0" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state remote_node="true" id="rabbitmq-bundle-0" uname="rabbitmq-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <lrm id="rabbitmq-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="27:10:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;27:10:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="15" rc-code="0" op-status="0" interval="0" last-run="1540983906" last-rc-change="1540983906" exec-time="16479" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/>
++ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="26:13:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" transition-magic="0:0;26:13:0:62f6c4c6-db69-4b50-b399-dd240c0fd7a1" exit-reason="" on_node="controller-0" call-id="52" rc-code="0" op-status="0" interval="10000" last-rc-change="1540983978" exec-time="5548" queue-time="0" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ </status>
++</cib>
+--
+1.8.3.1
+
diff --git a/SPECS/pacemaker.spec b/SPECS/pacemaker.spec
index 6a47ba2..03fa12e 100644
--- a/SPECS/pacemaker.spec
+++ b/SPECS/pacemaker.spec
@@ -160,7 +160,7 @@
Name: pacemaker
Summary: Scalable High-Availability cluster resource manager
Version: %{pcmkversion}
-Release: %{pcmk_release}%{?dist}
+Release: %{pcmk_release}%{?dist}.1
%if %{defined _unitdir}
License: GPLv2+ and LGPLv2+
%else
@@ -183,6 +183,9 @@ Patch4: 004-cleanup.patch
Patch5: 005-corosync.patch
Patch6: 006-fail-count.patch
Patch7: 007-stderr.patch
+Patch8: 008-bundle-ordering.patch
+Patch9: 009-sbd-guest.patch
+Patch10: 010-route-notify.patch
# patches that aren't from upstream
Patch100: lrmd-protocol-version.patch
@@ -867,6 +870,14 @@ exit 0
%attr(0644,root,root) %{_datadir}/pacemaker/nagios/plugins-metadata/*
%changelog
+* Tue Nov 6 2018 Ken Gaillot <kgaillot@redhat.com> - 1.1.19-8.1
+- Route remote clone notifications through correct cluster node
+- Allow clean-up of managed guest nodes and bundle nodes
+- Allow use of guest nodes and bundles in clusters with sbd fencing
+- Resolves: rhbz#1646347
+- Resolves: rhbz#1646350
+- Resolves: rhbz#1646872
+
* Mon Sep 24 2018 Ken Gaillot <kgaillot@redhat.com> - 1.1.19-8
- Ensure crm_resource --force-* commands get stderr messages
- Resolves: rhbz#1628947