diff --git a/SOURCES/011-notifs.patch b/SOURCES/011-notifs.patch
new file mode 100644
index 0000000..d67c6c5
--- /dev/null
+++ b/SOURCES/011-notifs.patch
@@ -0,0 +1,2352 @@
+From 141de63f1ddf48dc3c27f8397e0058e21bdecf46 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Mon, 26 Nov 2018 15:45:56 -0600
+Subject: [PATCH 1/5] Fix: scheduler: don't send clone notifications to a
+ stopped remote node
+
+Since b3f9a5bbb, we discard faked executor results when resource information is
+unavailable. This has exposed pre-existing issues where clone notifications were
+mistakenly scheduled for Pacemaker Remote nodes. Previously, the cluster node
+that had hosted the Pacemaker Remote connection would fake the result, and the
+transition would proceed. Now, if the cluster node doesn't happen to have the
+resource information cached, the result will not be sent, and thus the
+transition will get an action timeout. This permanently blocks later actions in
+the transition.
+
+This commit avoids such a situation where start and promote clone notifications
+were scheduled for a clone instance on a Pacemaker Remote node whose remote
+connection is stopping, and thus would be stopped by the time the notification
+would be needed.
+
+This is slightly modified from a patch provided by Andrew Beekhof
+<andrew@beekhof.net>.
+
+RHBZ#1652752
+---
+ pengine/notif.c | 36 ++++++++++++++++++++++++++++++++++++
+ 1 file changed, 36 insertions(+)
+
+diff --git a/pengine/notif.c b/pengine/notif.c
+index 4913249..cdc382d 100644
+--- a/pengine/notif.c
++++ b/pengine/notif.c
+@@ -631,6 +631,28 @@ expand_notification_data(resource_t *rsc, notify_data_t * n_data, pe_working_set
+ return required;
+ }
+
++/*
++ * \internal
++ * \brief Find any remote connection start relevant to an action
++ *
++ * \param[in] action Action to chek
++ *
++ * \return If action is behind a remote connection, connection's start
++ */
++static pe_action_t *
++find_remote_start(pe_action_t *action)
++{
++ if (action && action->node) {
++ pe_resource_t *remote_rsc = action->node->details->remote_rsc;
++
++ if (remote_rsc) {
++ return find_first_action(remote_rsc->actions, NULL, RSC_START,
++ NULL);
++ }
++ }
++ return NULL;
++}
++
+ void
+ create_notifications(resource_t * rsc, notify_data_t * n_data, pe_working_set_t * data_set)
+ {
+@@ -738,6 +760,20 @@ create_notifications(resource_t * rsc, notify_data_t * n_data, pe_working_set_t
+ rsc->id);
+
+ } else if (task == start_rsc || task == action_promote) {
++
++ if (start) {
++ pe_action_t *remote_start = find_remote_start(start);
++
++ if (remote_start
++ && is_not_set(remote_start->flags, pe_action_runnable)) {
++ /* Start and promote actions for a clone instance behind
++ * a Pacemaker Remote connection happen after the
++ * connection starts. If the connection start is blocked, do
++ * not schedule notifications for these actions.
++ */
++ return;
++ }
++ }
+ if (task != start_rsc || start == NULL || is_set(start->flags, pe_action_optional)) {
+ pe_notify(rsc, rsc->allocated_to, n_data->pre, n_data->pre_done, n_data, data_set);
+ }
+--
+1.8.3.1
+
+
+From 2793dcd9cfd20989d64ce1c553d63c28d9c7cb59 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Wed, 28 Nov 2018 18:12:35 -0600
+Subject: [PATCH 2/5] Test: scheduler: new test for clone notifications behind
+ a stopping remote
+
+Don't send notifications to a remote node whose connection has been stopped.
+---
+ pengine/regression.sh | 1 +
+ pengine/test10/notify-behind-stopping-remote.dot | 84 +++++
+ pengine/test10/notify-behind-stopping-remote.exp | 388 +++++++++++++++++++++
+ .../test10/notify-behind-stopping-remote.scores | 65 ++++
+ .../test10/notify-behind-stopping-remote.summary | 58 +++
+ pengine/test10/notify-behind-stopping-remote.xml | 187 ++++++++++
+ 6 files changed, 783 insertions(+)
+ create mode 100644 pengine/test10/notify-behind-stopping-remote.dot
+ create mode 100644 pengine/test10/notify-behind-stopping-remote.exp
+ create mode 100644 pengine/test10/notify-behind-stopping-remote.scores
+ create mode 100644 pengine/test10/notify-behind-stopping-remote.summary
+ create mode 100644 pengine/test10/notify-behind-stopping-remote.xml
+
+diff --git a/pengine/regression.sh b/pengine/regression.sh
+index fedd1b7..f719df6 100755
+--- a/pengine/regression.sh
++++ b/pengine/regression.sh
+@@ -484,6 +484,7 @@ do_test notify-3 "Notify move, confirm"
+ do_test novell-239079 "Notification priority"
+ #do_test notify-2 "Notify - 764"
+ do_test route-remote-notify "Route remote notify actions through correct cluster node"
++do_test notify-behind-stopping-remote "Don't schedule notifications behind stopped remote"
+
+ echo ""
+ do_test 594 "OSDL #594 - Unrunnable actions scheduled in transition"
+diff --git a/pengine/test10/notify-behind-stopping-remote.dot b/pengine/test10/notify-behind-stopping-remote.dot
+new file mode 100644
+index 0000000..cac4d5a
+--- /dev/null
++++ b/pengine/test10/notify-behind-stopping-remote.dot
+@@ -0,0 +1,84 @@
++digraph "g" {
++"Cancel redis_monitor_45000 redis-bundle-0" -> "redis_promote_0 redis-bundle-0" [ style = bold]
++"Cancel redis_monitor_45000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"Cancel redis_monitor_60000 redis-bundle-0" -> "redis_promote_0 redis-bundle-0" [ style = bold]
++"Cancel redis_monitor_60000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"all_stopped" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-1_monitor_60000 ra2" [ style=dashed color="red" fontcolor="black"]
++"redis-bundle-1_start_0 ra2" -> "redis-bundle-1_monitor_60000 ra2" [ style = dashed]
++"redis-bundle-1_start_0 ra2" -> "redis_monitor_45000 redis-bundle-1" [ style = dashed]
++"redis-bundle-1_start_0 ra2" -> "redis_monitor_60000 redis-bundle-1" [ style = dashed]
++"redis-bundle-1_start_0 ra2" -> "redis_start_0 redis-bundle-1" [ style = dashed]
++"redis-bundle-1_start_0 ra2" [ style=dashed color="red" fontcolor="black"]
++"redis-bundle-1_stop_0 ra2" -> "all_stopped" [ style = bold]
++"redis-bundle-1_stop_0 ra2" -> "redis-bundle-1_start_0 ra2" [ style = dashed]
++"redis-bundle-1_stop_0 ra2" -> "redis-bundle-docker-1_stop_0 ra2" [ style = bold]
++"redis-bundle-1_stop_0 ra2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-1_stop_0 ra2" -> "all_stopped" [ style = bold]
++"redis-bundle-docker-1_stop_0 ra2" -> "redis-bundle_stopped_0" [ style = bold]
++"redis-bundle-docker-1_stop_0 ra2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis-bundle_promoted_0" [ style = bold]
++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold]
++"redis-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle-master_pre_notify_promote_0" [ style = bold]
++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle_running_0" [ style = bold]
++"redis-bundle-master_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold]
++"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_promote_0" [ style = bold]
++"redis-bundle-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold]
++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_start_0" [ style = bold]
++"redis-bundle-master_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_post_notify_promoted_0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
++"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-2" [ style = bold]
++"redis-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_post_notify_running_0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold]
++"redis-bundle-master_post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_pre_notify_promote_0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
++"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-2" [ style = bold]
++"redis-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_pre_notify_start_0" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = bold]
++"redis-bundle-master_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_promoted_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold]
++"redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_running_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold]
++"redis-bundle-master_running_0" -> "redis-bundle-master_promote_0" [ style = bold]
++"redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold]
++"redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-1" [ style = dashed]
++"redis-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold]
++"redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_running_0" -> "redis-bundle_promote_0" [ style = bold]
++"redis-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold]
++"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_stop_0" -> "redis-bundle-docker-1_stop_0 ra2" [ style = bold]
++"redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_stopped_0" -> "redis-bundle_promote_0" [ style = bold]
++"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold]
++"redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis_monitor_45000 redis-bundle-1" [ style=dashed color="red" fontcolor="black"]
++"redis_monitor_60000 redis-bundle-1" [ style=dashed color="red" fontcolor="black"]
++"redis_post_notify_promoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
++"redis_post_notify_promoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis_post_notify_promoted_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
++"redis_post_notify_promoted_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
++"redis_pre_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
++"redis_pre_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis_pre_notify_promote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
++"redis_pre_notify_promote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
++"redis_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold]
++"redis_promote_0 redis-bundle-0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold]
++"redis_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis_start_0 redis-bundle-1" -> "redis-bundle-master_running_0" [ style = dashed]
++"redis_start_0 redis-bundle-1" -> "redis_monitor_45000 redis-bundle-1" [ style = dashed]
++"redis_start_0 redis-bundle-1" -> "redis_monitor_60000 redis-bundle-1" [ style = dashed]
++"redis_start_0 redis-bundle-1" [ style=dashed color="red" fontcolor="black"]
++}
+diff --git a/pengine/test10/notify-behind-stopping-remote.exp b/pengine/test10/notify-behind-stopping-remote.exp
+new file mode 100644
+index 0000000..44e6356
+--- /dev/null
++++ b/pengine/test10/notify-behind-stopping-remote.exp
+@@ -0,0 +1,388 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
++ <synapse id="0" priority="1000000">
++ <action_set>
++ <rsc_op id="77" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="ra1 ra2 ra3 ra1 ra2 ra3" CRM_meta_notify_all_uname="ra1 ra2 ra3 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 ra3 ra2 ra1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="ra1" CRM_meta_timeout="90000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="52" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="1">
++ <action_set>
++ <rsc_op id="76" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="ra1 ra2 ra3 ra1 ra2 ra3" CRM_meta_notify_all_uname="ra1 ra2 ra3 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 ra3 ra2 ra1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="ra1" CRM_meta_timeout="90000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="50" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="2">
++ <action_set>
++ <rsc_op id="30" operation="monitor" operation_key="redis_monitor_20000" internal_operation_key="redis:0_monitor_20000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_op_target_rc="8" CRM_meta_physical_host="ra1" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="29" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="53" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_promoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="3">
++ <action_set>
++ <rsc_op id="29" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="ra1 ra2 ra3 ra1 ra2 ra3" CRM_meta_notify_all_uname="ra1 ra2 ra3 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 ra3 ra2 ra1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="ra1" CRM_meta_timeout="120000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="7" operation="cancel" operation_key="redis_monitor_60000" internal_operation_key="redis:0_monitor_60000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="8" operation="cancel" operation_key="redis_monitor_45000" internal_operation_key="redis:0_monitor_45000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="48" operation="promote" operation_key="redis-bundle-master_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="4">
++ <action_set>
++ <rsc_op id="8" operation="cancel" operation_key="redis_monitor_45000" internal_operation_key="redis:0_monitor_45000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="45000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_operation="monitor" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="5">
++ <action_set>
++ <rsc_op id="7" operation="cancel" operation_key="redis_monitor_60000" internal_operation_key="redis:0_monitor_60000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_operation="monitor" CRM_meta_role="Slave" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="6" priority="1000000">
++ <action_set>
++ <rsc_op id="79" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="ra3">
++ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="ra1 ra2 ra3 ra1 ra2 ra3" CRM_meta_notify_all_uname="ra1 ra2 ra3 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 ra3 ra2 ra1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="ra3" CRM_meta_timeout="90000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="52" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7">
++ <action_set>
++ <rsc_op id="78" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="ra3">
++ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="ra1 ra2 ra3 ra1 ra2 ra3" CRM_meta_notify_all_uname="ra1 ra2 ra3 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 ra3 ra2 ra1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="ra3" CRM_meta_timeout="90000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="50" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="8" priority="1000000">
++ <action_set>
++ <pseudo_event id="53" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_promoted_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="promote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="52" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="77" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="79" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="ra3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="9" priority="1000000">
++ <action_set>
++ <pseudo_event id="52" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="promote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="49" operation="promoted" operation_key="redis-bundle-master_promoted_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="51" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="10">
++ <action_set>
++ <pseudo_event id="51" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_promote_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="promote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="50" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="76" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="78" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="ra3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="11">
++ <action_set>
++ <pseudo_event id="50" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="promote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="41" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="12" priority="1000000">
++ <action_set>
++ <pseudo_event id="49" operation="promoted" operation_key="redis-bundle-master_promoted_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="29" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="13">
++ <action_set>
++ <pseudo_event id="48" operation="promote" operation_key="redis-bundle-master_promote_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="37" operation="running" operation_key="redis-bundle-master_running_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="51" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_promote_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="60" operation="promote" operation_key="redis-bundle_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="14" priority="1000000">
++ <action_set>
++ <pseudo_event id="41" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_running_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="40" operation="notify" operation_key="redis-bundle-master_post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="15" priority="1000000">
++ <action_set>
++ <pseudo_event id="40" operation="notify" operation_key="redis-bundle-master_post_notify_running_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="37" operation="running" operation_key="redis-bundle-master_running_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="39" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="16">
++ <action_set>
++ <pseudo_event id="39" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_start_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="38" operation="notify" operation_key="redis-bundle-master_pre_notify_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="17">
++ <action_set>
++ <pseudo_event id="38" operation="notify" operation_key="redis-bundle-master_pre_notify_start_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="18" priority="1000000">
++ <action_set>
++ <pseudo_event id="37" operation="running" operation_key="redis-bundle-master_running_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="36" operation="start" operation_key="redis-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="19">
++ <action_set>
++ <pseudo_event id="36" operation="start" operation_key="redis-bundle-master_start_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="23" operation="start" operation_key="redis-bundle_start_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="39" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="20">
++ <action_set>
++ <rsc_op id="16" operation="stop" operation_key="redis-bundle-docker-1_stop_0" on_node="ra2" on_node_uuid="2">
++ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="ra2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="docker.io/tripleoqueens/centos-binary-redis:current-tripleo-rdo" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/usr/sbin/pacemaker_remoted" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3123 -v /dev/log:/dev/log -v /dev/zero:/etc/libqb/force-filesystem-sockets:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/redis.conf:/etc/redis.conf:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /usr/lib/ocf:/usr/lib/ocf:rw -v /var/log/pacemaker:/var/log/pacemaker:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log --user=root --log-driver=journald "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="17" operation="stop" operation_key="redis-bundle-1_stop_0" on_node="ra2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="25" operation="stop" operation_key="redis-bundle_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="21">
++ <action_set>
++ <rsc_op id="17" operation="stop" operation_key="redis-bundle-1_stop_0" on_node="ra2" on_node_uuid="2">
++ <primitive id="redis-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-1" CRM_meta_on_node="ra2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="ra2" port="3123"/>
++ <downed>
++ <node id="redis-bundle-1"/>
++ </downed>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="22" priority="1000000">
++ <action_set>
++ <pseudo_event id="61" operation="promoted" operation_key="redis-bundle_promoted_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="53" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_promoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="23">
++ <action_set>
++ <pseudo_event id="60" operation="promote" operation_key="redis-bundle_promote_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="24" operation="running" operation_key="redis-bundle_running_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="26" operation="stopped" operation_key="redis-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="24" priority="1000000">
++ <action_set>
++ <pseudo_event id="26" operation="stopped" operation_key="redis-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="16" operation="stop" operation_key="redis-bundle-docker-1_stop_0" on_node="ra2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="25">
++ <action_set>
++ <pseudo_event id="25" operation="stop" operation_key="redis-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="26" priority="1000000">
++ <action_set>
++ <pseudo_event id="24" operation="running" operation_key="redis-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="41" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="27">
++ <action_set>
++ <pseudo_event id="23" operation="start" operation_key="redis-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="26" operation="stopped" operation_key="redis-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="28">
++ <action_set>
++ <pseudo_event id="11" operation="all_stopped" operation_key="all_stopped">
++ <attributes />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="16" operation="stop" operation_key="redis-bundle-docker-1_stop_0" on_node="ra2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="17" operation="stop" operation_key="redis-bundle-1_stop_0" on_node="ra2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++</transition_graph>
+diff --git a/pengine/test10/notify-behind-stopping-remote.scores b/pengine/test10/notify-behind-stopping-remote.scores
+new file mode 100644
+index 0000000..e58b614
+--- /dev/null
++++ b/pengine/test10/notify-behind-stopping-remote.scores
+@@ -0,0 +1,65 @@
++Allocation scores:
++Using the original execution date of: 2018-11-22 20:36:07Z
++clone_color: redis-bundle-master allocation score on ra1: -INFINITY
++clone_color: redis-bundle-master allocation score on ra2: -INFINITY
++clone_color: redis-bundle-master allocation score on ra3: -INFINITY
++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
++clone_color: redis-bundle-master allocation score on redis-bundle-1: 0
++clone_color: redis-bundle-master allocation score on redis-bundle-2: 0
++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
++clone_color: redis:1 allocation score on redis-bundle-1: INFINITY
++clone_color: redis:2 allocation score on redis-bundle-2: INFINITY
++container_color: redis-bundle allocation score on ra1: 0
++container_color: redis-bundle allocation score on ra2: -INFINITY
++container_color: redis-bundle allocation score on ra3: 0
++container_color: redis-bundle-0 allocation score on ra1: 0
++container_color: redis-bundle-0 allocation score on ra2: 0
++container_color: redis-bundle-0 allocation score on ra3: 0
++container_color: redis-bundle-1 allocation score on ra1: 0
++container_color: redis-bundle-1 allocation score on ra2: 0
++container_color: redis-bundle-1 allocation score on ra3: 0
++container_color: redis-bundle-2 allocation score on ra1: 0
++container_color: redis-bundle-2 allocation score on ra2: 0
++container_color: redis-bundle-2 allocation score on ra3: 0
++container_color: redis-bundle-docker-0 allocation score on ra1: 0
++container_color: redis-bundle-docker-0 allocation score on ra2: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on ra3: 0
++container_color: redis-bundle-docker-1 allocation score on ra1: 0
++container_color: redis-bundle-docker-1 allocation score on ra2: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on ra3: 0
++container_color: redis-bundle-docker-2 allocation score on ra1: 0
++container_color: redis-bundle-docker-2 allocation score on ra2: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on ra3: 0
++container_color: redis-bundle-master allocation score on ra1: 0
++container_color: redis-bundle-master allocation score on ra2: 0
++container_color: redis-bundle-master allocation score on ra3: 0
++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-2: -INFINITY
++container_color: redis:0 allocation score on redis-bundle-0: 501
++container_color: redis:1 allocation score on redis-bundle-1: 500
++container_color: redis:2 allocation score on redis-bundle-2: 501
++native_color: redis-bundle-0 allocation score on ra1: 10000
++native_color: redis-bundle-0 allocation score on ra2: 0
++native_color: redis-bundle-0 allocation score on ra3: 0
++native_color: redis-bundle-1 allocation score on ra1: 0
++native_color: redis-bundle-1 allocation score on ra2: 0
++native_color: redis-bundle-1 allocation score on ra3: 0
++native_color: redis-bundle-2 allocation score on ra1: 0
++native_color: redis-bundle-2 allocation score on ra2: 0
++native_color: redis-bundle-2 allocation score on ra3: 10000
++native_color: redis-bundle-docker-0 allocation score on ra1: 0
++native_color: redis-bundle-docker-0 allocation score on ra2: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on ra3: 0
++native_color: redis-bundle-docker-1 allocation score on ra1: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on ra2: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on ra3: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on ra1: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on ra2: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on ra3: 0
++native_color: redis:0 allocation score on redis-bundle-0: INFINITY
++native_color: redis:1 allocation score on redis-bundle-1: INFINITY
++native_color: redis:2 allocation score on redis-bundle-2: INFINITY
++redis:0 promotion score on redis-bundle-0: 1
++redis:1 promotion score on redis-bundle-1: -1
++redis:2 promotion score on redis-bundle-2: 1
+diff --git a/pengine/test10/notify-behind-stopping-remote.summary b/pengine/test10/notify-behind-stopping-remote.summary
+new file mode 100644
+index 0000000..b9342b9
+--- /dev/null
++++ b/pengine/test10/notify-behind-stopping-remote.summary
+@@ -0,0 +1,58 @@
++Using the original execution date of: 2018-11-22 20:36:07Z
++
++Current cluster status:
++Online: [ ra1 ra2 ra3 ]
++Containers: [ redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ]
++
++ Docker container set: redis-bundle [docker.io/tripleoqueens/centos-binary-redis:current-tripleo-rdo]
++ redis-bundle-0 (ocf::heartbeat:redis): Slave ra1
++ redis-bundle-1 (ocf::heartbeat:redis): Stopped ra2
++ redis-bundle-2 (ocf::heartbeat:redis): Slave ra3
++
++Transition Summary:
++ * Promote redis:0 ( Slave -> Master redis-bundle-0 )
++ * Stop redis-bundle-docker-1 ( ra2 ) due to node availability
++ * Stop redis-bundle-1 ( ra2 ) due to unrunnable redis-bundle-docker-1 start
++ * Start redis:1 ( redis-bundle-1 ) due to unrunnable redis-bundle-docker-1 start (blocked)
++
++Executing cluster transition:
++ * Resource action: redis cancel=45000 on redis-bundle-0
++ * Resource action: redis cancel=60000 on redis-bundle-0
++ * Pseudo action: redis-bundle-master_pre_notify_start_0
++ * Resource action: redis-bundle-1 stop on ra2
++ * Pseudo action: redis-bundle_stop_0
++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_start_0
++ * Resource action: redis-bundle-docker-1 stop on ra2
++ * Pseudo action: redis-bundle_stopped_0
++ * Pseudo action: redis-bundle_start_0
++ * Pseudo action: all_stopped
++ * Pseudo action: redis-bundle-master_start_0
++ * Pseudo action: redis-bundle-master_running_0
++ * Pseudo action: redis-bundle-master_post_notify_running_0
++ * Pseudo action: redis-bundle-master_confirmed-post_notify_running_0
++ * Pseudo action: redis-bundle_running_0
++ * Pseudo action: redis-bundle-master_pre_notify_promote_0
++ * Pseudo action: redis-bundle_promote_0
++ * Resource action: redis notify on redis-bundle-0
++ * Resource action: redis notify on redis-bundle-2
++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_promote_0
++ * Pseudo action: redis-bundle-master_promote_0
++ * Resource action: redis promote on redis-bundle-0
++ * Pseudo action: redis-bundle-master_promoted_0
++ * Pseudo action: redis-bundle-master_post_notify_promoted_0
++ * Resource action: redis notify on redis-bundle-0
++ * Resource action: redis notify on redis-bundle-2
++ * Pseudo action: redis-bundle-master_confirmed-post_notify_promoted_0
++ * Pseudo action: redis-bundle_promoted_0
++ * Resource action: redis monitor=20000 on redis-bundle-0
++Using the original execution date of: 2018-11-22 20:36:07Z
++
++Revised cluster status:
++Online: [ ra1 ra2 ra3 ]
++Containers: [ redis-bundle-0:redis-bundle-docker-0 redis-bundle-2:redis-bundle-docker-2 ]
++
++ Docker container set: redis-bundle [docker.io/tripleoqueens/centos-binary-redis:current-tripleo-rdo]
++ redis-bundle-0 (ocf::heartbeat:redis): Master ra1
++ redis-bundle-1 (ocf::heartbeat:redis): Stopped
++ redis-bundle-2 (ocf::heartbeat:redis): Slave ra3
++
+diff --git a/pengine/test10/notify-behind-stopping-remote.xml b/pengine/test10/notify-behind-stopping-remote.xml
+new file mode 100644
+index 0000000..66351b8
+--- /dev/null
++++ b/pengine/test10/notify-behind-stopping-remote.xml
+@@ -0,0 +1,187 @@
++<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.10" epoch="53" num_updates="5" admin_epoch="0" cib-last-written="Thu Nov 22 15:35:52 2018" update-origin="ra2" update-client="crm_resource" update-user="root" have-quorum="1" dc-uuid="3" execution-date="1542918967">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.19-8.el7_6.1-c3c624ea3d"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="ratester"/>
++ <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1542909574"/>
++ </cluster_property_set>
++ <cluster_property_set id="redis_replication">
++ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="ra2"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="ra1"/>
++ <node id="2" uname="ra2"/>
++ <node id="3" uname="ra3"/>
++ </nodes>
++ <resources>
++ <bundle id="redis-bundle">
++ <docker masters="1" network="host" replicas="3" image="docker.io/tripleoqueens/centos-binary-redis:current-tripleo-rdo" run-command="/usr/sbin/pacemaker_remoted" options="--user=root --log-driver=journald"/>
++ <network control-port="3123"/>
++ <storage>
++ <storage-mapping target-dir="/dev/log" id="map0" source-dir="/dev/log"/>
++ <storage-mapping target-dir="/etc/libqb/force-filesystem-sockets" options="ro" id="map1" source-dir="/dev/zero"/>
++ <storage-mapping target-dir="/etc/hosts" options="ro" id="map2" source-dir="/etc/hosts"/>
++ <storage-mapping target-dir="/etc/localtime" options="ro" id="map3" source-dir="/etc/localtime"/>
++ <storage-mapping target-dir="/etc/redis.conf" options="ro" id="map4" source-dir="/etc/redis.conf"/>
++ <storage-mapping target-dir="/var/lib/redis" options="rw" id="map5" source-dir="/var/lib/redis"/>
++ <storage-mapping target-dir="/var/log/redis" options="rw" id="map6" source-dir="/var/log/redis"/>
++ <storage-mapping target-dir="/var/run/redis" options="rw" id="map7" source-dir="/var/run/redis"/>
++ <storage-mapping target-dir="/usr/lib/ocf" options="rw" id="map8" source-dir="/usr/lib/ocf"/>
++ <storage-mapping target-dir="/var/log/pacemaker" options="rw" id="pcmk1" source-dir="/var/log/pacemaker"/>
++ </storage>
++ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
++ <instance_attributes id="redis-instance_attributes">
++ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
++ </instance_attributes>
++ <meta_attributes id="redis-meta_attributes">
++ <nvpair id="redis-meta_attributes-container-attribute-target" name="container-attribute-target" value="host"/>
++ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120s"/>
++ <op id="redis-monitor-interval-45s" interval="45s" name="monitor" timeout="60s"/>
++ <op id="redis-monitor-interval-20s" interval="20s" name="monitor" role="Master" timeout="60s"/>
++ <op id="redis-monitor-interval-60s" interval="60s" name="monitor" role="Slave" timeout="60s"/>
++ <op id="redis-notify-interval-0s" interval="0s" name="notify" timeout="90s"/>
++ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120s"/>
++ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="120s"/>
++ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ </resources>
++ <constraints>
++ <rsc_location id="cli-ban-redis-bundle-on-ra2" rsc="redis-bundle" role="Started" node="ra2" score="-INFINITY"/>
++ </constraints>
++ </configuration>
++ <status>
++ <node_state id="2" uname="ra2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="2">
++ <lrm_resources>
++ <lrm_resource id="redis-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:38:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;7:38:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="53" rc-code="7" op-status="0" interval="0" last-run="1542907811" last-rc-change="1542907811" exec-time="65" queue-time="0" op-digest="317386a3af010667314d88ddf93a0150"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;7:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1542907815" last-rc-change="1542907815" exec-time="122" queue-time="0" op-digest="cabda3f80e7220806c7022b88efab9d4"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="6:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;6:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="81" rc-code="0" op-status="0" interval="0" last-run="1542918685" last-rc-change="1542918685" exec-time="1479" queue-time="0" op-digest="ad90cb2f61c71c84e840c796e44d628a"/>
++ <lrm_rsc_op id="redis-bundle-docker-1_monitor_60000" operation_key="redis-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;7:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="82" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918686" exec-time="173" queue-time="0" op-digest="51d9043cab411f1c983d9539d2997c26"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-0" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-0">
++ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="5:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;5:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="109c7a28bbb65c3c294a591f03377dd3" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-1" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-1">
++ <lrm_rsc_op id="redis-bundle-1_last_0" operation_key="redis-bundle-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="8:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;8:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1542918686" last-rc-change="1542918686" exec-time="0" queue-time="0" op-digest="9c100f1c3496f07f7a953f9cc8fbd746" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="redis-bundle-1_monitor_60000" operation_key="redis-bundle-1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="17:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;17:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="14" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918687" exec-time="0" queue-time="0" op-digest="21306bcef84a89e55ae729d77ceca1dc"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-2" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-2">
++ <lrm_rsc_op id="redis-bundle-2_last_0" operation_key="redis-bundle-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;7:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="6ed7cb9eb2c67afe77784971250638a7" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="2">
++ <instance_attributes id="status-2"/>
++ </transient_attributes>
++ </node_state>
++ <node_state id="1" uname="ra1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="1">
++ <lrm_resources>
++ <lrm_resource id="redis-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="4:37:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;4:37:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="57" rc-code="7" op-status="0" interval="0" last-run="1542907811" last-rc-change="1542907811" exec-time="63" queue-time="0" op-digest="0160bbff6f3f012b38edc9bddb631132"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="2:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;2:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="84" rc-code="0" op-status="0" interval="0" last-run="1542918685" last-rc-change="1542918685" exec-time="1504" queue-time="0" op-digest="cabda3f80e7220806c7022b88efab9d4"/>
++ <lrm_rsc_op id="redis-bundle-docker-0_monitor_60000" operation_key="redis-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="3:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;3:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="85" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918686" exec-time="167" queue-time="0" op-digest="922138cf0d1c7127c35f2e174c4af461"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="4:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;4:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="68" rc-code="7" op-status="0" interval="0" last-run="1542907815" last-rc-change="1542907815" exec-time="132" queue-time="0" op-digest="ad90cb2f61c71c84e840c796e44d628a"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-0" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-0">
++ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="4:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;4:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1542918686" last-rc-change="1542918686" exec-time="0" queue-time="0" op-digest="109c7a28bbb65c3c294a591f03377dd3" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="redis-bundle-0_monitor_60000" operation_key="redis-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="12:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;12:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="14" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918687" exec-time="0" queue-time="0" op-digest="478c16eefb4fa513134f527efb7e063a"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-1" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-1">
++ <lrm_rsc_op id="redis-bundle-1_last_0" operation_key="redis-bundle-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="3:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;3:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="9c100f1c3496f07f7a953f9cc8fbd746" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-2" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-2">
++ <lrm_rsc_op id="redis-bundle-2_last_0" operation_key="redis-bundle-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="4:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;4:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="6ed7cb9eb2c67afe77784971250638a7" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="1">
++ <instance_attributes id="status-1">
++ <nvpair id="status-1-master-redis" name="master-redis" value="1"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ <node_state id="3" uname="ra3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="3">
++ <lrm_resources>
++ <lrm_resource id="redis-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="10:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;10:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="77" rc-code="0" op-status="0" interval="0" last-run="1542918685" last-rc-change="1542918685" exec-time="1501" queue-time="0" op-digest="317386a3af010667314d88ddf93a0150"/>
++ <lrm_rsc_op id="redis-bundle-docker-2_monitor_60000" operation_key="redis-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="11:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;11:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="78" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918687" exec-time="165" queue-time="0" op-digest="b4fcc691b4f687c09bb3dd0610b658ca"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="12:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;12:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="57" rc-code="7" op-status="0" interval="0" last-run="1542907815" last-rc-change="1542907815" exec-time="129" queue-time="0" op-digest="cabda3f80e7220806c7022b88efab9d4"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="14:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;14:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1542907815" last-rc-change="1542907815" exec-time="111" queue-time="0" op-digest="ad90cb2f61c71c84e840c796e44d628a"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-0" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-0">
++ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="8:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;8:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="109c7a28bbb65c3c294a591f03377dd3" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-1" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-1">
++ <lrm_rsc_op id="redis-bundle-1_last_0" operation_key="redis-bundle-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="9:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;9:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="9c100f1c3496f07f7a953f9cc8fbd746" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-2" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-2">
++ <lrm_rsc_op id="redis-bundle-2_last_0" operation_key="redis-bundle-2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="12:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;12:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1542918687" last-rc-change="1542918687" exec-time="0" queue-time="0" op-digest="6ed7cb9eb2c67afe77784971250638a7" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="redis-bundle-2_monitor_60000" operation_key="redis-bundle-2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="22:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;22:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="14" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918688" exec-time="0" queue-time="0" op-digest="6815f4f1d3f361f086d1e45371d9cb53"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="3">
++ <instance_attributes id="status-3">
++ <nvpair id="status-3-master-redis" name="master-redis" value="1"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ <node_state remote_node="true" id="redis-bundle-0" uname="redis-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <lrm id="redis-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis_last_0" operation_key="redis_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="27:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;27:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="8" rc-code="0" op-status="0" interval="0" last-run="1542918688" last-rc-change="1542918688" exec-time="5253" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ <lrm_rsc_op id="redis_monitor_60000" operation_key="redis_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="32:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;32:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="78" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918713" exec-time="1195" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ <lrm_rsc_op id="redis_monitor_45000" operation_key="redis_monitor_45000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="31:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;31:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="79" rc-code="0" op-status="0" interval="45000" last-rc-change="1542918715" exec-time="1191" queue-time="1114" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state remote_node="true" id="redis-bundle-1" uname="redis-bundle-1" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <lrm id="redis-bundle-1">
++ <lrm_resources>
++ <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis_last_0" operation_key="redis_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="30:148:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;30:148:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="143" rc-code="0" op-status="0" interval="0" last-run="1542918963" last-rc-change="1542918963" exec-time="2398" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="28:145:8:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:8;28:145:8:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="59" rc-code="8" op-status="0" interval="20000" last-rc-change="1542918715" exec-time="1210" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state remote_node="true" id="redis-bundle-2" uname="redis-bundle-2" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <lrm id="redis-bundle-2">
++ <lrm_resources>
++ <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis_last_0" operation_key="redis_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="32:144:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;32:144:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="8" rc-code="0" op-status="0" interval="0" last-run="1542918700" last-rc-change="1542918700" exec-time="5173" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ <lrm_rsc_op id="redis_monitor_60000" operation_key="redis_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="36:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;36:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="77" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918714" exec-time="1184" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ <lrm_rsc_op id="redis_monitor_45000" operation_key="redis_monitor_45000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="35:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;35:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="78" rc-code="0" op-status="0" interval="45000" last-rc-change="1542918715" exec-time="1157" queue-time="1102" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ </status>
++</cib>
+--
+1.8.3.1
+
+
+From 90bed507285d23218617f0cd520d788ba246761e Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Wed, 28 Nov 2018 18:30:50 -0600
+Subject: [PATCH 3/5] Test: scheduler: update bundle-order-fencing test for
+ notification change
+
+Don't schedule clone notifications on a Pacemaker Remote node that
+has just been fenced.
+---
+ pengine/test10/bundle-order-fencing.dot | 8 -
+ pengine/test10/bundle-order-fencing.exp | 246 ++++++++++++----------------
+ pengine/test10/bundle-order-fencing.summary | 2 -
+ 3 files changed, 107 insertions(+), 149 deletions(-)
+
+diff --git a/pengine/test10/bundle-order-fencing.dot b/pengine/test10/bundle-order-fencing.dot
+index e53a062..1e2721b 100644
+--- a/pengine/test10/bundle-order-fencing.dot
++++ b/pengine/test10/bundle-order-fencing.dot
+@@ -210,8 +210,6 @@ digraph "g" {
+ "redis-bundle-master_confirmed-post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"]
+ "redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis-bundle_promoted_0" [ style = bold]
+ "redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold]
+-"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed]
+-"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed]
+ "redis-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]
+ "redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle-master_pre_notify_promote_0" [ style = bold]
+ "redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle_running_0" [ style = bold]
+@@ -247,7 +245,6 @@ digraph "g" {
+ "redis-bundle-master_post_notify_demoted_0" -> "redis_post_notify_demoted_0 redis-bundle-2" [ style = bold]
+ "redis-bundle-master_post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"]
+ "redis-bundle-master_post_notify_promoted_0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
+-"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-0" [ style = bold]
+ "redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-1" [ style = bold]
+ "redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-2" [ style = bold]
+ "redis-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]
+@@ -262,7 +259,6 @@ digraph "g" {
+ "redis-bundle-master_pre_notify_demote_0" -> "redis_pre_notify_demote_0 redis-bundle-2" [ style = bold]
+ "redis-bundle-master_pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"]
+ "redis-bundle-master_pre_notify_promote_0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
+-"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-0" [ style = bold]
+ "redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-1" [ style = bold]
+ "redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-2" [ style = bold]
+ "redis-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"]
+@@ -325,8 +321,6 @@ digraph "g" {
+ "redis_post_notify_demoted_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
+ "redis_post_notify_demoted_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold]
+ "redis_post_notify_demoted_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
+-"redis_post_notify_promoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
+-"redis_post_notify_promoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+ "redis_post_notify_promoted_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
+ "redis_post_notify_promoted_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
+ "redis_post_notify_promoted_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
+@@ -345,8 +339,6 @@ digraph "g" {
+ "redis_pre_notify_demote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
+ "redis_pre_notify_demote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold]
+ "redis_pre_notify_demote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
+-"redis_pre_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
+-"redis_pre_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+ "redis_pre_notify_promote_0 redis-bundle-1" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
+ "redis_pre_notify_promote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
+ "redis_pre_notify_promote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
+diff --git a/pengine/test10/bundle-order-fencing.exp b/pengine/test10/bundle-order-fencing.exp
+index 2b8f5cf..84bffaa 100644
+--- a/pengine/test10/bundle-order-fencing.exp
++++ b/pengine/test10/bundle-order-fencing.exp
+@@ -521,32 +521,6 @@
+ </synapse>
+ <synapse id="39" priority="1000000">
+ <action_set>
+- <rsc_op id="240" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1">
+- <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+- <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+- </rsc_op>
+- </action_set>
+- <inputs>
+- <trigger>
+- <pseudo_event id="161" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
+- </trigger>
+- </inputs>
+- </synapse>
+- <synapse id="40">
+- <action_set>
+- <rsc_op id="239" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1">
+- <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+- <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+- </rsc_op>
+- </action_set>
+- <inputs>
+- <trigger>
+- <pseudo_event id="159" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
+- </trigger>
+- </inputs>
+- </synapse>
+- <synapse id="41" priority="1000000">
+- <action_set>
+ <pseudo_event id="216" operation="notified" operation_key="redis_notified_0" internal_operation_key="redis:0_confirmed-post_notify_stonith_0">
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stonith" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </pseudo_event>
+@@ -563,7 +537,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="42" priority="1000000">
++ <synapse id="40" priority="1000000">
+ <action_set>
+ <pseudo_event id="215" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:0_post_notify_stonith_0">
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stonith" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+@@ -575,7 +549,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="43">
++ <synapse id="41">
+ <action_set>
+ <pseudo_event id="141" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0">
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_timeout="200000" wait_last_known_master="true"/>
+@@ -593,7 +567,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="44">
++ <synapse id="42">
+ <action_set>
+ <pseudo_event id="140" operation="demote" operation_key="redis_demote_0" internal_operation_key="redis:0_demote_0">
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_timeout="120000" wait_last_known_master="true"/>
+@@ -605,9 +579,9 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="45" priority="1000000">
++ <synapse id="43" priority="1000000">
+ <action_set>
+- <rsc_op id="246" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="244" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -618,9 +592,9 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="46">
++ <synapse id="44">
+ <action_set>
+- <rsc_op id="245" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="243" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -631,9 +605,9 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="47" priority="1000000">
++ <synapse id="45" priority="1000000">
+ <action_set>
+- <rsc_op id="242" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="240" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -644,9 +618,9 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="48">
++ <synapse id="46">
+ <action_set>
+- <rsc_op id="241" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
++ <rsc_op id="239" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -657,7 +631,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="49">
++ <synapse id="47">
+ <action_set>
+ <rsc_op id="237" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:1_pre_notify_stop_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+@@ -670,7 +644,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="50" priority="1000000">
++ <synapse id="48" priority="1000000">
+ <action_set>
+ <rsc_op id="217" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:1_post_notify_stop_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+@@ -686,7 +660,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="51">
++ <synapse id="49">
+ <action_set>
+ <rsc_op id="137" operation="monitor" operation_key="redis_monitor_20000" internal_operation_key="redis:1_monitor_20000" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+@@ -711,7 +685,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="52">
++ <synapse id="50">
+ <action_set>
+ <rsc_op id="136" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:1_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+@@ -733,7 +707,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="53">
++ <synapse id="51">
+ <action_set>
+ <rsc_op id="28" operation="cancel" operation_key="redis_monitor_45000" internal_operation_key="redis:1_monitor_45000" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+@@ -742,7 +716,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="54">
++ <synapse id="52">
+ <action_set>
+ <rsc_op id="27" operation="cancel" operation_key="redis_monitor_60000" internal_operation_key="redis:1_monitor_60000" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
+@@ -751,9 +725,9 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="55" priority="1000000">
++ <synapse id="53" priority="1000000">
+ <action_set>
+- <rsc_op id="248" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="246" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -764,9 +738,9 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="56">
++ <synapse id="54">
+ <action_set>
+- <rsc_op id="247" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="245" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -777,9 +751,9 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="57" priority="1000000">
++ <synapse id="55" priority="1000000">
+ <action_set>
+- <rsc_op id="244" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="242" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -790,9 +764,9 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="58">
++ <synapse id="56">
+ <action_set>
+- <rsc_op id="243" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
++ <rsc_op id="241" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
+ </rsc_op>
+@@ -803,7 +777,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="59">
++ <synapse id="57">
+ <action_set>
+ <rsc_op id="238" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:2_pre_notify_stop_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+@@ -816,7 +790,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="60" priority="1000000">
++ <synapse id="58" priority="1000000">
+ <action_set>
+ <rsc_op id="218" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:2_post_notify_stop_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+@@ -832,7 +806,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="61" priority="1000000">
++ <synapse id="59" priority="1000000">
+ <action_set>
+ <pseudo_event id="168" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_demoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="demote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+@@ -843,14 +817,14 @@
+ <pseudo_event id="167" operation="notify" operation_key="redis-bundle-master_post_notify_demoted_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="246" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
++ <rsc_op id="244" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="248" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="246" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="62" priority="1000000">
++ <synapse id="60" priority="1000000">
+ <action_set>
+ <pseudo_event id="167" operation="notify" operation_key="redis-bundle-master_post_notify_demoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="demote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+@@ -865,7 +839,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="63">
++ <synapse id="61">
+ <action_set>
+ <pseudo_event id="166" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_demote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="demote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+@@ -876,14 +850,14 @@
+ <pseudo_event id="165" operation="notify" operation_key="redis-bundle-master_pre_notify_demote_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="245" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
++ <rsc_op id="243" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="247" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="245" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="64">
++ <synapse id="62">
+ <action_set>
+ <pseudo_event id="165" operation="notify" operation_key="redis-bundle-master_pre_notify_demote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="demote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+@@ -891,7 +865,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="65" priority="1000000">
++ <synapse id="63" priority="1000000">
+ <action_set>
+ <pseudo_event id="164" operation="demoted" operation_key="redis-bundle-master_demoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+@@ -906,7 +880,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="66">
++ <synapse id="64">
+ <action_set>
+ <pseudo_event id="163" operation="demote" operation_key="redis-bundle-master_demote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+@@ -921,7 +895,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="67" priority="1000000">
++ <synapse id="65" priority="1000000">
+ <action_set>
+ <pseudo_event id="162" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_promoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="promote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+@@ -932,17 +906,14 @@
+ <pseudo_event id="161" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="240" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1"/>
++ <rsc_op id="240" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="242" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+- </trigger>
+- <trigger>
+- <rsc_op id="244" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="242" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="68" priority="1000000">
++ <synapse id="66" priority="1000000">
+ <action_set>
+ <pseudo_event id="161" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="promote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+@@ -957,7 +928,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="69">
++ <synapse id="67">
+ <action_set>
+ <pseudo_event id="160" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_promote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="promote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+@@ -968,17 +939,14 @@
+ <pseudo_event id="159" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="239" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1"/>
+- </trigger>
+- <trigger>
+- <rsc_op id="241" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
++ <rsc_op id="239" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="243" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
++ <rsc_op id="241" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="70">
++ <synapse id="68">
+ <action_set>
+ <pseudo_event id="159" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="promote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+@@ -996,7 +964,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="71" priority="1000000">
++ <synapse id="69" priority="1000000">
+ <action_set>
+ <pseudo_event id="158" operation="promoted" operation_key="redis-bundle-master_promoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+@@ -1008,7 +976,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="72">
++ <synapse id="70">
+ <action_set>
+ <pseudo_event id="157" operation="promote" operation_key="redis-bundle-master_promote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+@@ -1032,7 +1000,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="73" priority="1000000">
++ <synapse id="71" priority="1000000">
+ <action_set>
+ <pseudo_event id="156" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_stopped_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+@@ -1050,7 +1018,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="74" priority="1000000">
++ <synapse id="72" priority="1000000">
+ <action_set>
+ <pseudo_event id="155" operation="notify" operation_key="redis-bundle-master_post_notify_stopped_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+@@ -1065,7 +1033,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="75">
++ <synapse id="73">
+ <action_set>
+ <pseudo_event id="154" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_stop_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+@@ -1083,7 +1051,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="76">
++ <synapse id="74">
+ <action_set>
+ <pseudo_event id="153" operation="notify" operation_key="redis-bundle-master_pre_notify_stop_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+@@ -1095,7 +1063,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="77" priority="1000000">
++ <synapse id="75" priority="1000000">
+ <action_set>
+ <pseudo_event id="152" operation="stopped" operation_key="redis-bundle-master_stopped_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+@@ -1110,7 +1078,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="78">
++ <synapse id="76">
+ <action_set>
+ <pseudo_event id="151" operation="stop" operation_key="redis-bundle-master_stop_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+@@ -1131,7 +1099,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="79" priority="1000000">
++ <synapse id="77" priority="1000000">
+ <action_set>
+ <pseudo_event id="150" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+@@ -1143,7 +1111,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="80" priority="1000000">
++ <synapse id="78" priority="1000000">
+ <action_set>
+ <pseudo_event id="149" operation="notify" operation_key="redis-bundle-master_post_notify_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+@@ -1158,7 +1126,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="81">
++ <synapse id="79">
+ <action_set>
+ <pseudo_event id="148" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+@@ -1170,7 +1138,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="82">
++ <synapse id="80">
+ <action_set>
+ <pseudo_event id="147" operation="notify" operation_key="redis-bundle-master_pre_notify_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+@@ -1185,7 +1153,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="83" priority="1000000">
++ <synapse id="81" priority="1000000">
+ <action_set>
+ <pseudo_event id="146" operation="running" operation_key="redis-bundle-master_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+@@ -1197,7 +1165,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="84">
++ <synapse id="82">
+ <action_set>
+ <pseudo_event id="145" operation="start" operation_key="redis-bundle-master_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+@@ -1218,7 +1186,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="85">
++ <synapse id="83">
+ <action_set>
+ <pseudo_event id="118" operation="stop" operation_key="redis-bundle-docker-0_stop_0">
+ <attributes CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/rhosp12/openstack-redis-docker:pcmklatest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/redis/:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+@@ -1236,7 +1204,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="86">
++ <synapse id="84">
+ <action_set>
+ <pseudo_event id="119" operation="stop" operation_key="redis-bundle-0_stop_0">
+ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_timeout="20000" addr="controller-0" port="3124"/>
+@@ -1244,7 +1212,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="87">
++ <synapse id="85">
+ <action_set>
+ <rsc_op id="40" operation="monitor" operation_key="redis-bundle-0_monitor_0" on_node="controller-2" on_node_uuid="3">
+ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+@@ -1253,7 +1221,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="88">
++ <synapse id="86">
+ <action_set>
+ <rsc_op id="34" operation="monitor" operation_key="redis-bundle-0_monitor_0" on_node="controller-1" on_node_uuid="2">
+ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
+@@ -1262,7 +1230,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="89">
++ <synapse id="87">
+ <action_set>
+ <rsc_op id="41" operation="monitor" operation_key="redis-bundle-1_monitor_0" on_node="controller-2" on_node_uuid="3">
+ <primitive id="redis-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
+@@ -1271,7 +1239,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="90">
++ <synapse id="88">
+ <action_set>
+ <rsc_op id="35" operation="monitor" operation_key="redis-bundle-2_monitor_0" on_node="controller-1" on_node_uuid="2">
+ <primitive id="redis-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
+@@ -1280,7 +1248,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="91">
++ <synapse id="89">
+ <action_set>
+ <rsc_op id="175" operation="monitor" operation_key="ip-192.168.24.7_monitor_10000" on_node="controller-2" on_node_uuid="3">
+ <primitive id="ip-192.168.24.7" class="ocf" provider="heartbeat" type="IPaddr2"/>
+@@ -1293,7 +1261,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="92">
++ <synapse id="90">
+ <action_set>
+ <rsc_op id="174" operation="start" operation_key="ip-192.168.24.7_start_0" on_node="controller-2" on_node_uuid="3">
+ <primitive id="ip-192.168.24.7" class="ocf" provider="heartbeat" type="IPaddr2"/>
+@@ -1309,7 +1277,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="93">
++ <synapse id="91">
+ <action_set>
+ <pseudo_event id="173" operation="stop" operation_key="ip-192.168.24.7_stop_0">
+ <attributes CRM_meta_name="stop" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.24.7"/>
+@@ -1324,7 +1292,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="94">
++ <synapse id="92">
+ <action_set>
+ <rsc_op id="178" operation="monitor" operation_key="ip-10.0.0.109_monitor_10000" on_node="controller-1" on_node_uuid="2">
+ <primitive id="ip-10.0.0.109" class="ocf" provider="heartbeat" type="IPaddr2"/>
+@@ -1337,7 +1305,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="95">
++ <synapse id="93">
+ <action_set>
+ <rsc_op id="177" operation="start" operation_key="ip-10.0.0.109_start_0" on_node="controller-1" on_node_uuid="2">
+ <primitive id="ip-10.0.0.109" class="ocf" provider="heartbeat" type="IPaddr2"/>
+@@ -1353,7 +1321,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="96">
++ <synapse id="94">
+ <action_set>
+ <pseudo_event id="176" operation="stop" operation_key="ip-10.0.0.109_stop_0">
+ <attributes CRM_meta_name="stop" CRM_meta_timeout="20000" cidr_netmask="32" ip="10.0.0.109"/>
+@@ -1368,7 +1336,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="97">
++ <synapse id="95">
+ <action_set>
+ <rsc_op id="187" operation="monitor" operation_key="ip-172.17.4.11_monitor_10000" on_node="controller-1" on_node_uuid="2">
+ <primitive id="ip-172.17.4.11" class="ocf" provider="heartbeat" type="IPaddr2"/>
+@@ -1381,7 +1349,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="98">
++ <synapse id="96">
+ <action_set>
+ <rsc_op id="186" operation="start" operation_key="ip-172.17.4.11_start_0" on_node="controller-1" on_node_uuid="2">
+ <primitive id="ip-172.17.4.11" class="ocf" provider="heartbeat" type="IPaddr2"/>
+@@ -1397,7 +1365,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="99">
++ <synapse id="97">
+ <action_set>
+ <pseudo_event id="185" operation="stop" operation_key="ip-172.17.4.11_stop_0">
+ <attributes CRM_meta_name="stop" CRM_meta_timeout="20000" cidr_netmask="32" ip="172.17.4.11"/>
+@@ -1412,7 +1380,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="100">
++ <synapse id="98">
+ <action_set>
+ <pseudo_event id="188" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0">
+ <attributes CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/rhosp12/openstack-haproxy-docker:pcmklatest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/haproxy/:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
+@@ -1427,7 +1395,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="101">
++ <synapse id="99">
+ <action_set>
+ <rsc_op id="203" operation="monitor" operation_key="stonith-fence_ipmilan-5254003e8e97_monitor_60000" on_node="controller-1" on_node_uuid="2">
+ <primitive id="stonith-fence_ipmilan-5254003e8e97" class="stonith" type="fence_ipmilan"/>
+@@ -1440,7 +1408,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="102">
++ <synapse id="100">
+ <action_set>
+ <rsc_op id="202" operation="start" operation_key="stonith-fence_ipmilan-5254003e8e97_start_0" on_node="controller-1" on_node_uuid="2">
+ <primitive id="stonith-fence_ipmilan-5254003e8e97" class="stonith" type="fence_ipmilan"/>
+@@ -1453,7 +1421,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="103">
++ <synapse id="101">
+ <action_set>
+ <pseudo_event id="201" operation="stop" operation_key="stonith-fence_ipmilan-5254003e8e97_stop_0">
+ <attributes CRM_meta_timeout="20000" action="reboot" ipaddr="172.16.0.1" ipport="6231" lanplus="true" login="admin" passwd="password" pcmk_host_list="controller-2" privlvl="administrator"/>
+@@ -1461,7 +1429,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="104">
++ <synapse id="102">
+ <action_set>
+ <rsc_op id="206" operation="monitor" operation_key="stonith-fence_ipmilan-5254000dcb3f_monitor_60000" on_node="controller-2" on_node_uuid="3">
+ <primitive id="stonith-fence_ipmilan-5254000dcb3f" class="stonith" type="fence_ipmilan"/>
+@@ -1474,7 +1442,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="105">
++ <synapse id="103">
+ <action_set>
+ <rsc_op id="205" operation="start" operation_key="stonith-fence_ipmilan-5254000dcb3f_start_0" on_node="controller-2" on_node_uuid="3">
+ <primitive id="stonith-fence_ipmilan-5254000dcb3f" class="stonith" type="fence_ipmilan"/>
+@@ -1487,7 +1455,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="106">
++ <synapse id="104">
+ <action_set>
+ <pseudo_event id="204" operation="stop" operation_key="stonith-fence_ipmilan-5254000dcb3f_stop_0">
+ <attributes CRM_meta_timeout="20000" action="reboot" ipaddr="172.16.0.1" ipport="6230" lanplus="true" login="admin" passwd="password" pcmk_host_list="controller-1" privlvl="administrator"/>
+@@ -1495,7 +1463,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="107">
++ <synapse id="105">
+ <action_set>
+ <pseudo_event id="214" operation="stonith" operation_key="stonith-redis-bundle-0-off" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0">
+ <attributes CRM_meta_master_redis="1" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_stonith_action="off" />
+@@ -1510,7 +1478,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="108">
++ <synapse id="106">
+ <action_set>
+ <pseudo_event id="209" operation="stonith" operation_key="stonith-rabbitmq-bundle-0-off" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0">
+ <attributes CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_rmq_node_attr_last_known_rabbitmq="rabbit@controller-0" CRM_meta_stonith_action="off" />
+@@ -1525,7 +1493,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="109">
++ <synapse id="107">
+ <action_set>
+ <pseudo_event id="208" operation="stonith" operation_key="stonith-galera-bundle-0-off" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0">
+ <attributes CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_stonith_action="off" />
+@@ -1540,7 +1508,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="110">
++ <synapse id="108">
+ <action_set>
+ <pseudo_event id="207" operation="stonith_complete" operation_key="stonith_complete">
+ <attributes />
+@@ -1561,7 +1529,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="111" priority="1000000">
++ <synapse id="109" priority="1000000">
+ <action_set>
+ <pseudo_event id="196" operation="stopped" operation_key="haproxy-bundle_stopped_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1573,7 +1541,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="112">
++ <synapse id="110">
+ <action_set>
+ <pseudo_event id="195" operation="stop" operation_key="haproxy-bundle_stop_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1581,7 +1549,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="113" priority="1000000">
++ <synapse id="111" priority="1000000">
+ <action_set>
+ <pseudo_event id="172" operation="demoted" operation_key="redis-bundle_demoted_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1596,7 +1564,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="114">
++ <synapse id="112">
+ <action_set>
+ <pseudo_event id="171" operation="demote" operation_key="redis-bundle_demote_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1604,7 +1572,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="115" priority="1000000">
++ <synapse id="113" priority="1000000">
+ <action_set>
+ <pseudo_event id="170" operation="promoted" operation_key="redis-bundle_promoted_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1616,7 +1584,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="116">
++ <synapse id="114">
+ <action_set>
+ <pseudo_event id="169" operation="promote" operation_key="redis-bundle_promote_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1634,7 +1602,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="117" priority="1000000">
++ <synapse id="115" priority="1000000">
+ <action_set>
+ <pseudo_event id="133" operation="stopped" operation_key="redis-bundle_stopped_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1649,7 +1617,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="118">
++ <synapse id="116">
+ <action_set>
+ <pseudo_event id="132" operation="stop" operation_key="redis-bundle_stop_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1661,7 +1629,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="119" priority="1000000">
++ <synapse id="117" priority="1000000">
+ <action_set>
+ <pseudo_event id="131" operation="running" operation_key="redis-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1673,7 +1641,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="120">
++ <synapse id="118">
+ <action_set>
+ <pseudo_event id="130" operation="start" operation_key="redis-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1688,7 +1656,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="121" priority="1000000">
++ <synapse id="119" priority="1000000">
+ <action_set>
+ <pseudo_event id="117" operation="demoted" operation_key="galera-bundle_demoted_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1703,7 +1671,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="122">
++ <synapse id="120">
+ <action_set>
+ <pseudo_event id="116" operation="demote" operation_key="galera-bundle_demote_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1711,7 +1679,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="123" priority="1000000">
++ <synapse id="121" priority="1000000">
+ <action_set>
+ <pseudo_event id="92" operation="stopped" operation_key="galera-bundle_stopped_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1726,7 +1694,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="124">
++ <synapse id="122">
+ <action_set>
+ <pseudo_event id="91" operation="stop" operation_key="galera-bundle_stop_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1738,7 +1706,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="125" priority="1000000">
++ <synapse id="123" priority="1000000">
+ <action_set>
+ <pseudo_event id="90" operation="running" operation_key="galera-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1750,7 +1718,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="126">
++ <synapse id="124">
+ <action_set>
+ <pseudo_event id="89" operation="start" operation_key="galera-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1765,7 +1733,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="127" priority="1000000">
++ <synapse id="125" priority="1000000">
+ <action_set>
+ <pseudo_event id="57" operation="stopped" operation_key="rabbitmq-bundle_stopped_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1780,7 +1748,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="128">
++ <synapse id="126">
+ <action_set>
+ <pseudo_event id="56" operation="stop" operation_key="rabbitmq-bundle_stop_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1788,7 +1756,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="129" priority="1000000">
++ <synapse id="127" priority="1000000">
+ <action_set>
+ <pseudo_event id="55" operation="running" operation_key="rabbitmq-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1800,7 +1768,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="130">
++ <synapse id="128">
+ <action_set>
+ <pseudo_event id="54" operation="start" operation_key="rabbitmq-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -1808,7 +1776,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="131">
++ <synapse id="129">
+ <action_set>
+ <pseudo_event id="29" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+@@ -1877,7 +1845,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="132">
++ <synapse id="130">
+ <action_set>
+ <crm_event id="1" operation="stonith" operation_key="stonith-controller-0-reboot" on_node="controller-0" on_node_uuid="1">
+ <attributes CRM_meta_cinder_volume_role="true" CRM_meta_galera_role="true" CRM_meta_haproxy_role="true" CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_rabbitmq_role="true" CRM_meta_redis_role="true" CRM_meta_rmq_node_attr_last_known_rabbitmq="rabbit@controller-0" CRM_meta_stonith_action="reboot" />
+diff --git a/pengine/test10/bundle-order-fencing.summary b/pengine/test10/bundle-order-fencing.summary
+index d398a12..a3dc3d4 100644
+--- a/pengine/test10/bundle-order-fencing.summary
++++ b/pengine/test10/bundle-order-fencing.summary
+@@ -174,7 +174,6 @@ Executing cluster transition:
+ * Pseudo action: redis-bundle_running_0
+ * Pseudo action: redis-bundle-master_pre_notify_promote_0
+ * Pseudo action: redis-bundle_promote_0
+- * Resource action: redis notify on redis-bundle-0
+ * Resource action: redis notify on redis-bundle-1
+ * Resource action: redis notify on redis-bundle-2
+ * Pseudo action: redis-bundle-master_confirmed-pre_notify_promote_0
+@@ -182,7 +181,6 @@ Executing cluster transition:
+ * Resource action: redis promote on redis-bundle-1
+ * Pseudo action: redis-bundle-master_promoted_0
+ * Pseudo action: redis-bundle-master_post_notify_promoted_0
+- * Resource action: redis notify on redis-bundle-0
+ * Resource action: redis notify on redis-bundle-1
+ * Resource action: redis notify on redis-bundle-2
+ * Pseudo action: redis-bundle-master_confirmed-post_notify_promoted_0
+--
+1.8.3.1
+
+
+From 48198ca839b62de1316d7ae6ab0994dedb37b523 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Tue, 27 Nov 2018 17:00:25 -0600
+Subject: [PATCH 4/5] Refactor: controller: make process_lrm_event() void
+
+All callers ignored the return value
+---
+ crmd/crmd_lrm.h | 3 ++-
+ crmd/lrm.c | 8 +++-----
+ 2 files changed, 5 insertions(+), 6 deletions(-)
+
+diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h
+index d115056..3e1596d 100644
+--- a/crmd/crmd_lrm.h
++++ b/crmd/crmd_lrm.h
+@@ -170,4 +170,5 @@ gboolean remote_ra_is_in_maintenance(lrm_state_t * lrm_state);
+ void remote_ra_process_maintenance_nodes(xmlNode *xml);
+ gboolean remote_ra_controlling_guest(lrm_state_t * lrm_state);
+
+-gboolean process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurring_op_s *pending);
++void process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
++ struct recurring_op_s *pending);
+diff --git a/crmd/lrm.c b/crmd/lrm.c
+index d18665c..5e5af9f 100644
+--- a/crmd/lrm.c
++++ b/crmd/lrm.c
+@@ -2515,7 +2515,7 @@ unescape_newlines(const char *string)
+ return ret;
+ }
+
+-gboolean
++void
+ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurring_op_s *pending)
+ {
+ char *op_id = NULL;
+@@ -2526,8 +2526,8 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
+ gboolean removed = FALSE;
+ lrmd_rsc_info_t *rsc = NULL;
+
+- CRM_CHECK(op != NULL, return FALSE);
+- CRM_CHECK(op->rsc_id != NULL, return FALSE);
++ CRM_CHECK(op != NULL, return);
++ CRM_CHECK(op->rsc_id != NULL, return);
+
+ op_id = make_stop_id(op->rsc_id, op->call_id);
+ op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
+@@ -2679,6 +2679,4 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
+ lrmd_free_rsc_info(rsc);
+ free(op_key);
+ free(op_id);
+-
+- return TRUE;
+ }
+--
+1.8.3.1
+
+
+From 77dd44e214401d4dd953a8bafa2469b36d70948e Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Tue, 27 Nov 2018 17:02:36 -0600
+Subject: [PATCH 5/5] Low: controller: be more tolerant of malformed executor
+ events
+
+b3f9a5bb was overzealous in discarding faked executor results without any
+resource information. Since that commit, synthesize_lrmd_failure() would check
+for resource information, and send a CIB update if the synthesized operation
+were recordable, but would otherwise (such as for notifications) discard the
+result.
+
+This means the fix was complete, because non-recordable actions for a
+resource behind a just-died remote connection would get lost. It also
+exposed two pre-existing bugs regarding notifications mis-scheduled on
+the wrong node. Any of these would block the transition from completing.
+
+Now, process_lrm_event() can handle missing lrm_state or resource information,
+so it can be called by synthesize_lrmd_failure() without any checking. This
+leads to all the normal handling for non-recordable operations, which doesn't
+require resource information. We log an assertion if the resource information
+is not found, so that we can still get some visibility into bugs. This won't
+be of use in the case of mis-scheduled notifications, but it could help in
+other situations.
+---
+ crmd/crmd_lrm.h | 2 +-
+ crmd/lrm.c | 148 ++++++++++++++++++++++++++++++--------------------
+ crmd/lrm_state.c | 2 +-
+ crmd/remote_lrmd_ra.c | 2 +-
+ 4 files changed, 93 insertions(+), 61 deletions(-)
+
+diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h
+index 3e1596d..0870817 100644
+--- a/crmd/crmd_lrm.h
++++ b/crmd/crmd_lrm.h
+@@ -171,4 +171,4 @@ void remote_ra_process_maintenance_nodes(xmlNode *xml);
+ gboolean remote_ra_controlling_guest(lrm_state_t * lrm_state);
+
+ void process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
+- struct recurring_op_s *pending);
++ struct recurring_op_s *pending, xmlNode *action_xml);
+diff --git a/crmd/lrm.c b/crmd/lrm.c
+index 5e5af9f..0d64f59 100644
+--- a/crmd/lrm.c
++++ b/crmd/lrm.c
+@@ -314,7 +314,7 @@ lrm_op_callback(lrmd_event_data_t * op)
+ lrm_state = lrm_state_find(nodename);
+ CRM_ASSERT(lrm_state != NULL);
+
+- process_lrm_event(lrm_state, op, NULL);
++ process_lrm_event(lrm_state, op, NULL, NULL);
+ }
+
+ /* A_LRM_CONNECT */
+@@ -1434,7 +1434,6 @@ static void
+ synthesize_lrmd_failure(lrm_state_t *lrm_state, xmlNode *action, int rc)
+ {
+ lrmd_event_data_t *op = NULL;
+- lrmd_rsc_info_t *rsc_info = NULL;
+ const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK);
+ const char *target_node = crm_element_value(action, XML_LRM_ATTR_TARGET);
+ xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE);
+@@ -1464,35 +1463,8 @@ synthesize_lrmd_failure(lrm_state_t *lrm_state, xmlNode *action, int rc)
+ crm_info("Faking %s_%s_%d result (%d) on %s",
+ op->rsc_id, op->op_type, op->interval, op->rc, target_node);
+
+- /* Process the result as if it came from the LRM, if possible
+- * (i.e. resource info can be obtained from the lrm_state).
+- */
+- if (lrm_state) {
+- rsc_info = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
+- }
+- if (rsc_info) {
+- lrmd_free_rsc_info(rsc_info);
+- process_lrm_event(lrm_state, op, NULL);
+-
+- } else if (controld_action_is_recordable(op->op_type)) {
+- /* If we can't process the result normally, at least write it to the CIB
+- * if possible, so the PE can act on it.
+- */
+- const char *standard = crm_element_value(xml_rsc, XML_AGENT_ATTR_CLASS);
+- const char *provider = crm_element_value(xml_rsc, XML_AGENT_ATTR_PROVIDER);
+- const char *type = crm_element_value(xml_rsc, XML_ATTR_TYPE);
+-
+- if (standard && type) {
+- rsc_info = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
+- do_update_resource(target_node, rsc_info, op);
+- lrmd_free_rsc_info(rsc_info);
+- } else {
+- // @TODO Should we direct ack?
+- crm_info("Can't fake %s failure (%d) on %s without resource standard and type",
+- crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc,
+- target_node);
+- }
+- }
++ // Process the result as if it came from the LRM
++ process_lrm_event(lrm_state, op, NULL, action);
+ lrmd_free_event(op);
+ }
+
+@@ -1555,7 +1527,7 @@ fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
+
+ if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
+ crm_info("Failing resource %s...", rsc->id);
+- process_lrm_event(lrm_state, op, NULL);
++ process_lrm_event(lrm_state, op, NULL, xml);
+ op->op_status = PCMK_LRM_OP_DONE;
+ op->rc = PCMK_OCF_OK;
+ lrmd_free_rsc_info(rsc);
+@@ -2315,7 +2287,7 @@ do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *operat
+ crm_err("Operation %s on resource %s failed to execute on remote node %s: %d",
+ operation, rsc->id, lrm_state->node_name, call_id);
+ fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_UNKNOWN_ERROR);
+- process_lrm_event(lrm_state, op, NULL);
++ process_lrm_event(lrm_state, op, NULL, NULL);
+
+ } else {
+ /* record all operations so we can wait
+@@ -2516,7 +2488,8 @@ unescape_newlines(const char *string)
+ }
+
+ void
+-process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurring_op_s *pending)
++process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
++ struct recurring_op_s *pending, xmlNode *action_xml)
+ {
+ char *op_id = NULL;
+ char *op_key = NULL;
+@@ -2525,16 +2498,49 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
+ gboolean remove = FALSE;
+ gboolean removed = FALSE;
+ lrmd_rsc_info_t *rsc = NULL;
++ const char *node_name = NULL;
+
+ CRM_CHECK(op != NULL, return);
+ CRM_CHECK(op->rsc_id != NULL, return);
+
+ op_id = make_stop_id(op->rsc_id, op->call_id);
+ op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
+- rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
++
++ // Get resource info if available (from executor state or action XML)
++ if (lrm_state) {
++ rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
++ }
++ if ((rsc == NULL) && action_xml) {
++ xmlNode *xml = find_xml_node(action_xml, XML_CIB_TAG_RESOURCE, TRUE);
++
++ const char *standard = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
++ const char *provider = crm_element_value(xml, XML_AGENT_ATTR_PROVIDER);
++ const char *type = crm_element_value(xml, XML_ATTR_TYPE);
++
++ if (standard && type) {
++ crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
++ op->rsc_id, standard,
++ (provider? ":" : ""), (provider? provider : ""), type);
++ rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
++ } else {
++ crm_err("Can't process %s result because %s agent information not cached or in XML",
++ op_key, op->rsc_id);
++ }
++ }
++ CRM_LOG_ASSERT(rsc != NULL); // If it's still NULL, there's a bug somewhere
++
++ // Get node name if available (from executor state or action XML)
++ if (lrm_state) {
++ node_name = lrm_state->node_name;
++ } else if (action_xml) {
++ node_name = crm_element_value(action_xml, XML_LRM_ATTR_TARGET);
++ }
++
+ if(pending == NULL) {
+ remove = TRUE;
+- pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
++ if (lrm_state) {
++ pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
++ }
+ }
+
+ if (op->op_status == PCMK_LRM_OP_ERROR) {
+@@ -2554,7 +2560,14 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
+
+ if (op->op_status != PCMK_LRM_OP_CANCELLED) {
+ if (controld_action_is_recordable(op->op_type)) {
+- update_id = do_update_resource(lrm_state->node_name, rsc, op);
++ if (node_name && rsc) {
++ update_id = do_update_resource(node_name, rsc, op);
++ } else {
++ // @TODO Should we direct ack?
++ crm_err("Unable to record %s result in CIB: %s",
++ op_key,
++ (node_name? "No resource information" : "No node name"));
++ }
+ } else {
+ send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
+ }
+@@ -2575,7 +2588,9 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
+
+ } else if (pending->remove) {
+ /* The tengine canceled this op, we have been waiting for the cancel to finish. */
+- erase_lrm_history_by_op(lrm_state, op);
++ if (lrm_state) {
++ erase_lrm_history_by_op(lrm_state, op);
++ }
+
+ } else if (op->rsc_deleted) {
+ /* The tengine initiated this op, but it was cancelled outside of the
+@@ -2595,14 +2610,23 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
+ /* The caller will do this afterwards, but keep the logging consistent */
+ removed = TRUE;
+
+- } else if ((op->interval == 0) && g_hash_table_remove(lrm_state->pending_ops, op_id)) {
+- removed = TRUE;
+- crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
+- op_key, op->call_id, op_id, g_hash_table_size(lrm_state->pending_ops));
++ } else if (lrm_state && ((op->interval == 0)
++ || (op->op_status == PCMK_LRM_OP_CANCELLED))) {
+
+- } else if(op->interval != 0 && op->op_status == PCMK_LRM_OP_CANCELLED) {
+- removed = TRUE;
+- g_hash_table_remove(lrm_state->pending_ops, op_id);
++ gboolean found = g_hash_table_remove(lrm_state->pending_ops, op_id);
++
++ if (op->interval != 0) {
++ removed = TRUE;
++ } else if (found) {
++ removed = TRUE;
++ crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
++ op_key, op->call_id, op_id,
++ g_hash_table_size(lrm_state->pending_ops));
++ }
++ }
++
++ if (node_name == NULL) {
++ node_name = "unknown node"; // for logging
+ }
+
+ switch (op->op_status) {
+@@ -2610,7 +2634,7 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
+ crm_info("Result of %s operation for %s on %s: %s "
+ CRM_XS " call=%d key=%s confirmed=%s",
+ crm_action_str(op->op_type, op->interval),
+- op->rsc_id, lrm_state->node_name,
++ op->rsc_id, node_name,
+ services_lrm_status_str(op->op_status),
+ op->call_id, op_key, (removed? "true" : "false"));
+ break;
+@@ -2620,7 +2644,7 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
+ "Result of %s operation for %s on %s: %d (%s) "
+ CRM_XS " call=%d key=%s confirmed=%s cib-update=%d",
+ crm_action_str(op->op_type, op->interval),
+- op->rsc_id, lrm_state->node_name,
++ op->rsc_id, node_name,
+ op->rc, services_ocf_exitcode_str(op->rc),
+ op->call_id, op_key, (removed? "true" : "false"),
+ update_id);
+@@ -2630,7 +2654,7 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
+ crm_err("Result of %s operation for %s on %s: %s "
+ CRM_XS " call=%d key=%s timeout=%dms",
+ crm_action_str(op->op_type, op->interval),
+- op->rsc_id, lrm_state->node_name,
++ op->rsc_id, node_name,
+ services_lrm_status_str(op->op_status),
+ op->call_id, op_key, op->timeout);
+ break;
+@@ -2639,14 +2663,16 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
+ crm_err("Result of %s operation for %s on %s: %s "
+ CRM_XS " call=%d key=%s confirmed=%s status=%d cib-update=%d",
+ crm_action_str(op->op_type, op->interval),
+- op->rsc_id, lrm_state->node_name,
++ op->rsc_id, node_name,
+ services_lrm_status_str(op->op_status), op->call_id, op_key,
+ (removed? "true" : "false"), op->op_status, update_id);
+ }
+
+ if (op->output) {
+ char *prefix =
+- crm_strdup_printf("%s-%s_%s_%d:%d", lrm_state->node_name, op->rsc_id, op->op_type, op->interval, op->call_id);
++ crm_strdup_printf("%s-%s_%s_%d:%d", node_name,
++ op->rsc_id, op->op_type, op->interval,
++ op->call_id);
+
+ if (op->rc) {
+ crm_log_output(LOG_NOTICE, prefix, op->output);
+@@ -2656,25 +2682,31 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
+ free(prefix);
+ }
+
+- if (safe_str_neq(op->op_type, RSC_METADATA)) {
+- crmd_alert_resource_op(lrm_state->node_name, op);
+- } else if (op->rc == PCMK_OCF_OK) {
+- char *metadata = unescape_newlines(op->output);
++ if (lrm_state) {
++ if (safe_str_neq(op->op_type, RSC_METADATA)) {
++ crmd_alert_resource_op(lrm_state->node_name, op);
++ } else if (rsc && (op->rc == PCMK_OCF_OK)) {
++ char *metadata = unescape_newlines(op->output);
+
+- metadata_cache_update(lrm_state->metadata_cache, rsc, metadata);
+- free(metadata);
++ metadata_cache_update(lrm_state->metadata_cache, rsc, metadata);
++ free(metadata);
++ }
+ }
+
+ if (op->rsc_deleted) {
+ crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
+- delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
++ if (lrm_state) {
++ delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
++ }
+ }
+
+ /* If a shutdown was escalated while operations were pending,
+ * then the FSA will be stalled right now... allow it to continue
+ */
+ mainloop_set_trigger(fsa_source);
+- update_history_cache(lrm_state, rsc, op);
++ if (lrm_state && rsc) {
++ update_history_cache(lrm_state, rsc, op);
++ }
+
+ lrmd_free_rsc_info(rsc);
+ free(op_key);
+diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c
+index 40da762..d8a0039 100644
+--- a/crmd/lrm_state.c
++++ b/crmd/lrm_state.c
+@@ -96,7 +96,7 @@ fail_pending_op(gpointer key, gpointer value, gpointer user_data)
+ event.remote_nodename = lrm_state->node_name;
+ event.params = op->params;
+
+- process_lrm_event(lrm_state, &event, op);
++ process_lrm_event(lrm_state, &event, op, NULL);
+ return TRUE;
+ }
+
+diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c
+index 6fa05f6..2d04588 100644
+--- a/crmd/remote_lrmd_ra.c
++++ b/crmd/remote_lrmd_ra.c
+@@ -519,7 +519,7 @@ synthesize_lrmd_success(lrm_state_t *lrm_state, const char *rsc_id, const char *
+ op.t_run = time(NULL);
+ op.t_rcchange = op.t_run;
+ op.call_id = generate_callid();
+- process_lrm_event(lrm_state, &op, NULL);
++ process_lrm_event(lrm_state, &op, NULL, NULL);
+ }
+
+ void
+--
+1.8.3.1
+
diff --git a/SPECS/pacemaker.spec b/SPECS/pacemaker.spec
index 0b73a07..b4ed85d 100644
--- a/SPECS/pacemaker.spec
+++ b/SPECS/pacemaker.spec
@@ -160,7 +160,7 @@
Name: pacemaker
Summary: Scalable High-Availability cluster resource manager
Version: %{pcmkversion}
-Release: %{pcmk_release}%{?dist}.1
+Release: %{pcmk_release}%{?dist}.2
%if %{defined _unitdir}
License: GPLv2+ and LGPLv2+
%else
@@ -186,6 +186,7 @@ Patch7: 007-stderr.patch
Patch8: 008-bundle-ordering.patch
Patch9: 009-sbd-guest.patch
Patch10: 010-route-notify.patch
+Patch11: 011-notifs.patch
# patches that aren't from upstream
Patch100: lrmd-protocol-version.patch
@@ -205,7 +206,7 @@ Provides: pcmk-cluster-manager
%{?systemd_requires}
-ExclusiveArch: aarch64 i686 ppc64le s390x x86_64 %{arm}
+ExclusiveArch: aarch64 i686 ppc64le s390x x86_64
# Pacemaker targets compatibility with python 2.6+ and 3.2+
Requires: python >= 2.6
@@ -870,6 +871,10 @@ exit 0
%attr(0644,root,root) %{_datadir}/pacemaker/nagios/plugins-metadata/*
%changelog
+* Thu Nov 29 2018 Ken Gaillot <kgaillot@redhat.com> - 1.1.19-8.2
+- Don't schedule clone notifications behind stopped remote connection
+- Resolves: rhbz#1654602
+
* Tue Nov 6 2018 Ken Gaillot <kgaillot@redhat.com> - 1.1.19-8.1
- Route remote clone notifications through correct cluster node
- Allow clean-up of managed guest nodes and bundle nodes