From 141de63f1ddf48dc3c27f8397e0058e21bdecf46 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Mon, 26 Nov 2018 15:45:56 -0600
Subject: [PATCH 1/5] Fix: scheduler: don't send clone notifications to a
stopped remote node
Since b3f9a5bbb, we discard faked executor results when resource information is
unavailable. This has exposed pre-existing issues where clone notifications were
mistakenly scheduled for Pacemaker Remote nodes. Previously, the cluster node
that had hosted the Pacemaker Remote connection would fake the result, and the
transition would proceed. Now, if the cluster node doesn't happen to have the
resource information cached, the result will not be sent, and thus the
transition will get an action timeout. This permanently blocks later actions in
the transition.
This commit avoids such a situation where start and promote clone notifications
were scheduled for a clone instance on a Pacemaker Remote node whose remote
connection is stopping, and thus would be stopped by the time the notification
would be needed.
This is slightly modified from a patch provided by Andrew Beekhof
<andrew@beekhof.net>.
RHBZ#1652752
---
pengine/notif.c | 36 ++++++++++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/pengine/notif.c b/pengine/notif.c
index 4913249..cdc382d 100644
--- a/pengine/notif.c
+++ b/pengine/notif.c
@@ -631,6 +631,28 @@ expand_notification_data(resource_t *rsc, notify_data_t * n_data, pe_working_set
return required;
}
+/*
+ * \internal
+ * \brief Find any remote connection start relevant to an action
+ *
+ * \param[in] action Action to chek
+ *
+ * \return If action is behind a remote connection, connection's start
+ */
+static pe_action_t *
+find_remote_start(pe_action_t *action)
+{
+ if (action && action->node) {
+ pe_resource_t *remote_rsc = action->node->details->remote_rsc;
+
+ if (remote_rsc) {
+ return find_first_action(remote_rsc->actions, NULL, RSC_START,
+ NULL);
+ }
+ }
+ return NULL;
+}
+
void
create_notifications(resource_t * rsc, notify_data_t * n_data, pe_working_set_t * data_set)
{
@@ -738,6 +760,20 @@ create_notifications(resource_t * rsc, notify_data_t * n_data, pe_working_set_t
rsc->id);
} else if (task == start_rsc || task == action_promote) {
+
+ if (start) {
+ pe_action_t *remote_start = find_remote_start(start);
+
+ if (remote_start
+ && is_not_set(remote_start->flags, pe_action_runnable)) {
+ /* Start and promote actions for a clone instance behind
+ * a Pacemaker Remote connection happen after the
+ * connection starts. If the connection start is blocked, do
+ * not schedule notifications for these actions.
+ */
+ return;
+ }
+ }
if (task != start_rsc || start == NULL || is_set(start->flags, pe_action_optional)) {
pe_notify(rsc, rsc->allocated_to, n_data->pre, n_data->pre_done, n_data, data_set);
}
--
1.8.3.1
From 2793dcd9cfd20989d64ce1c553d63c28d9c7cb59 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Wed, 28 Nov 2018 18:12:35 -0600
Subject: [PATCH 2/5] Test: scheduler: new test for clone notifications behind
a stopping remote
Don't send notifications to a remote node whose connection has been stopped.
---
pengine/regression.sh | 1 +
pengine/test10/notify-behind-stopping-remote.dot | 84 +++++
pengine/test10/notify-behind-stopping-remote.exp | 388 +++++++++++++++++++++
.../test10/notify-behind-stopping-remote.scores | 65 ++++
.../test10/notify-behind-stopping-remote.summary | 58 +++
pengine/test10/notify-behind-stopping-remote.xml | 187 ++++++++++
6 files changed, 783 insertions(+)
create mode 100644 pengine/test10/notify-behind-stopping-remote.dot
create mode 100644 pengine/test10/notify-behind-stopping-remote.exp
create mode 100644 pengine/test10/notify-behind-stopping-remote.scores
create mode 100644 pengine/test10/notify-behind-stopping-remote.summary
create mode 100644 pengine/test10/notify-behind-stopping-remote.xml
diff --git a/pengine/regression.sh b/pengine/regression.sh
index fedd1b7..f719df6 100755
--- a/pengine/regression.sh
+++ b/pengine/regression.sh
@@ -484,6 +484,7 @@ do_test notify-3 "Notify move, confirm"
do_test novell-239079 "Notification priority"
#do_test notify-2 "Notify - 764"
do_test route-remote-notify "Route remote notify actions through correct cluster node"
+do_test notify-behind-stopping-remote "Don't schedule notifications behind stopped remote"
echo ""
do_test 594 "OSDL #594 - Unrunnable actions scheduled in transition"
diff --git a/pengine/test10/notify-behind-stopping-remote.dot b/pengine/test10/notify-behind-stopping-remote.dot
new file mode 100644
index 0000000..cac4d5a
--- /dev/null
+++ b/pengine/test10/notify-behind-stopping-remote.dot
@@ -0,0 +1,84 @@
+digraph "g" {
+"Cancel redis_monitor_45000 redis-bundle-0" -> "redis_promote_0 redis-bundle-0" [ style = bold]
+"Cancel redis_monitor_45000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"Cancel redis_monitor_60000 redis-bundle-0" -> "redis_promote_0 redis-bundle-0" [ style = bold]
+"Cancel redis_monitor_60000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-1_monitor_60000 ra2" [ style=dashed color="red" fontcolor="black"]
+"redis-bundle-1_start_0 ra2" -> "redis-bundle-1_monitor_60000 ra2" [ style = dashed]
+"redis-bundle-1_start_0 ra2" -> "redis_monitor_45000 redis-bundle-1" [ style = dashed]
+"redis-bundle-1_start_0 ra2" -> "redis_monitor_60000 redis-bundle-1" [ style = dashed]
+"redis-bundle-1_start_0 ra2" -> "redis_start_0 redis-bundle-1" [ style = dashed]
+"redis-bundle-1_start_0 ra2" [ style=dashed color="red" fontcolor="black"]
+"redis-bundle-1_stop_0 ra2" -> "all_stopped" [ style = bold]
+"redis-bundle-1_stop_0 ra2" -> "redis-bundle-1_start_0 ra2" [ style = dashed]
+"redis-bundle-1_stop_0 ra2" -> "redis-bundle-docker-1_stop_0 ra2" [ style = bold]
+"redis-bundle-1_stop_0 ra2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-docker-1_stop_0 ra2" -> "all_stopped" [ style = bold]
+"redis-bundle-docker-1_stop_0 ra2" -> "redis-bundle_stopped_0" [ style = bold]
+"redis-bundle-docker-1_stop_0 ra2" [ style=bold color="green" fontcolor="black"]
+"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis-bundle_promoted_0" [ style = bold]
+"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold]
+"redis-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle-master_pre_notify_promote_0" [ style = bold]
+"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle_running_0" [ style = bold]
+"redis-bundle-master_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold]
+"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_promote_0" [ style = bold]
+"redis-bundle-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold]
+"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_start_0" [ style = bold]
+"redis-bundle-master_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_post_notify_promoted_0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
+"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-0" [ style = bold]
+"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-2" [ style = bold]
+"redis-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_post_notify_running_0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold]
+"redis-bundle-master_post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_pre_notify_promote_0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
+"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-0" [ style = bold]
+"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-2" [ style = bold]
+"redis-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_pre_notify_start_0" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = bold]
+"redis-bundle-master_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-0" [ style = bold]
+"redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_promoted_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold]
+"redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_running_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold]
+"redis-bundle-master_running_0" -> "redis-bundle-master_promote_0" [ style = bold]
+"redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold]
+"redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-1" [ style = dashed]
+"redis-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold]
+"redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_running_0" -> "redis-bundle_promote_0" [ style = bold]
+"redis-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold]
+"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_stop_0" -> "redis-bundle-docker-1_stop_0 ra2" [ style = bold]
+"redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+"redis-bundle_stopped_0" -> "redis-bundle_promote_0" [ style = bold]
+"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold]
+"redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"redis_monitor_45000 redis-bundle-1" [ style=dashed color="red" fontcolor="black"]
+"redis_monitor_60000 redis-bundle-1" [ style=dashed color="red" fontcolor="black"]
+"redis_post_notify_promoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
+"redis_post_notify_promoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"redis_post_notify_promoted_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
+"redis_post_notify_promoted_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
+"redis_pre_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
+"redis_pre_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"redis_pre_notify_promote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
+"redis_pre_notify_promote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
+"redis_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold]
+"redis_promote_0 redis-bundle-0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold]
+"redis_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
+"redis_start_0 redis-bundle-1" -> "redis-bundle-master_running_0" [ style = dashed]
+"redis_start_0 redis-bundle-1" -> "redis_monitor_45000 redis-bundle-1" [ style = dashed]
+"redis_start_0 redis-bundle-1" -> "redis_monitor_60000 redis-bundle-1" [ style = dashed]
+"redis_start_0 redis-bundle-1" [ style=dashed color="red" fontcolor="black"]
+}
diff --git a/pengine/test10/notify-behind-stopping-remote.exp b/pengine/test10/notify-behind-stopping-remote.exp
new file mode 100644
index 0000000..44e6356
--- /dev/null
+++ b/pengine/test10/notify-behind-stopping-remote.exp
@@ -0,0 +1,388 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0" priority="1000000">
+ <action_set>
+ <rsc_op id="77" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="ra1 ra2 ra3 ra1 ra2 ra3" CRM_meta_notify_all_uname="ra1 ra2 ra3 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 ra3 ra2 ra1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="ra1" CRM_meta_timeout="90000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="52" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="76" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="ra1 ra2 ra3 ra1 ra2 ra3" CRM_meta_notify_all_uname="ra1 ra2 ra3 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 ra3 ra2 ra1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="ra1" CRM_meta_timeout="90000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="50" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="30" operation="monitor" operation_key="redis_monitor_20000" internal_operation_key="redis:0_monitor_20000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_op_target_rc="8" CRM_meta_physical_host="ra1" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="29" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="53" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_promoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <rsc_op id="29" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="ra1 ra2 ra3 ra1 ra2 ra3" CRM_meta_notify_all_uname="ra1 ra2 ra3 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 ra3 ra2 ra1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_physical_host="ra1" CRM_meta_timeout="120000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="7" operation="cancel" operation_key="redis_monitor_60000" internal_operation_key="redis:0_monitor_60000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="8" operation="cancel" operation_key="redis_monitor_45000" internal_operation_key="redis:0_monitor_45000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="48" operation="promote" operation_key="redis-bundle-master_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <rsc_op id="8" operation="cancel" operation_key="redis_monitor_45000" internal_operation_key="redis:0_monitor_45000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="45000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_operation="monitor" CRM_meta_timeout="60000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <rsc_op id="7" operation="cancel" operation_key="redis_monitor_60000" internal_operation_key="redis:0_monitor_60000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_operation="monitor" CRM_meta_role="Slave" CRM_meta_timeout="60000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="6" priority="1000000">
+ <action_set>
+ <rsc_op id="79" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="ra3">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="ra1 ra2 ra3 ra1 ra2 ra3" CRM_meta_notify_all_uname="ra1 ra2 ra3 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 ra3 ra2 ra1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="ra3" CRM_meta_timeout="90000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="52" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <rsc_op id="78" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="ra3">
+ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="notify" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_all_hosts="ra1 ra2 ra3 ra1 ra2 ra3" CRM_meta_notify_all_uname="ra1 ra2 ra3 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 ra3 ra2 ra1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:1" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:0" CRM_meta_notify_promote_uname="redis-bundle-0" CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="redis-bundle-0 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="ra3" CRM_meta_timeout="90000" wait_last_known_master="true"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="50" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8" priority="1000000">
+ <action_set>
+ <pseudo_event id="53" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_promoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="promote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="52" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="77" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="79" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="ra3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="9" priority="1000000">
+ <action_set>
+ <pseudo_event id="52" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="promote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="49" operation="promoted" operation_key="redis-bundle-master_promoted_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="51" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="10">
+ <action_set>
+ <pseudo_event id="51" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_promote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="promote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="50" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="76" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="78" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="ra3"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="11">
+ <action_set>
+ <pseudo_event id="50" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="promote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="41" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="12" priority="1000000">
+ <action_set>
+ <pseudo_event id="49" operation="promoted" operation_key="redis-bundle-master_promoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="29" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="ra1"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="13">
+ <action_set>
+ <pseudo_event id="48" operation="promote" operation_key="redis-bundle-master_promote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="37" operation="running" operation_key="redis-bundle-master_running_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="51" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_promote_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="60" operation="promote" operation_key="redis-bundle_promote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="14" priority="1000000">
+ <action_set>
+ <pseudo_event id="41" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="40" operation="notify" operation_key="redis-bundle-master_post_notify_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="15" priority="1000000">
+ <action_set>
+ <pseudo_event id="40" operation="notify" operation_key="redis-bundle-master_post_notify_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="37" operation="running" operation_key="redis-bundle-master_running_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="39" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="16">
+ <action_set>
+ <pseudo_event id="39" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="38" operation="notify" operation_key="redis-bundle-master_pre_notify_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="17">
+ <action_set>
+ <pseudo_event id="38" operation="notify" operation_key="redis-bundle-master_pre_notify_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="18" priority="1000000">
+ <action_set>
+ <pseudo_event id="37" operation="running" operation_key="redis-bundle-master_running_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="36" operation="start" operation_key="redis-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="19">
+ <action_set>
+ <pseudo_event id="36" operation="start" operation_key="redis-bundle-master_start_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="23" operation="start" operation_key="redis-bundle_start_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="39" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="20">
+ <action_set>
+ <rsc_op id="16" operation="stop" operation_key="redis-bundle-docker-1_stop_0" on_node="ra2" on_node_uuid="2">
+ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
+ <attributes CRM_meta_on_node="ra2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="docker.io/tripleoqueens/centos-binary-redis:current-tripleo-rdo" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/usr/sbin/pacemaker_remoted" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3123 -v /dev/log:/dev/log -v /dev/zero:/etc/libqb/force-filesystem-sockets:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/redis.conf:/etc/redis.conf:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /usr/lib/ocf:/usr/lib/ocf:rw -v /var/log/pacemaker:/var/log/pacemaker:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log --user=root --log-driver=journald "/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="17" operation="stop" operation_key="redis-bundle-1_stop_0" on_node="ra2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="25" operation="stop" operation_key="redis-bundle_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="21">
+ <action_set>
+ <rsc_op id="17" operation="stop" operation_key="redis-bundle-1_stop_0" on_node="ra2" on_node_uuid="2">
+ <primitive id="redis-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
+ <attributes CRM_meta_container="redis-bundle-docker-1" CRM_meta_on_node="ra2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="ra2" port="3123"/>
+ <downed>
+ <node id="redis-bundle-1"/>
+ </downed>
+ </rsc_op>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="22" priority="1000000">
+ <action_set>
+ <pseudo_event id="61" operation="promoted" operation_key="redis-bundle_promoted_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="53" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_promoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="23">
+ <action_set>
+ <pseudo_event id="60" operation="promote" operation_key="redis-bundle_promote_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="24" operation="running" operation_key="redis-bundle_running_0"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="26" operation="stopped" operation_key="redis-bundle_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="24" priority="1000000">
+ <action_set>
+ <pseudo_event id="26" operation="stopped" operation_key="redis-bundle_stopped_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="16" operation="stop" operation_key="redis-bundle-docker-1_stop_0" on_node="ra2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="25">
+ <action_set>
+ <pseudo_event id="25" operation="stop" operation_key="redis-bundle_stop_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="26" priority="1000000">
+ <action_set>
+ <pseudo_event id="24" operation="running" operation_key="redis-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="41" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="27">
+ <action_set>
+ <pseudo_event id="23" operation="start" operation_key="redis-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="26" operation="stopped" operation_key="redis-bundle_stopped_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="28">
+ <action_set>
+ <pseudo_event id="11" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="16" operation="stop" operation_key="redis-bundle-docker-1_stop_0" on_node="ra2" on_node_uuid="2"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="17" operation="stop" operation_key="redis-bundle-1_stop_0" on_node="ra2" on_node_uuid="2"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
diff --git a/pengine/test10/notify-behind-stopping-remote.scores b/pengine/test10/notify-behind-stopping-remote.scores
new file mode 100644
index 0000000..e58b614
--- /dev/null
+++ b/pengine/test10/notify-behind-stopping-remote.scores
@@ -0,0 +1,65 @@
+Allocation scores:
+Using the original execution date of: 2018-11-22 20:36:07Z
+clone_color: redis-bundle-master allocation score on ra1: -INFINITY
+clone_color: redis-bundle-master allocation score on ra2: -INFINITY
+clone_color: redis-bundle-master allocation score on ra3: -INFINITY
+clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
+clone_color: redis-bundle-master allocation score on redis-bundle-1: 0
+clone_color: redis-bundle-master allocation score on redis-bundle-2: 0
+clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
+clone_color: redis:1 allocation score on redis-bundle-1: INFINITY
+clone_color: redis:2 allocation score on redis-bundle-2: INFINITY
+container_color: redis-bundle allocation score on ra1: 0
+container_color: redis-bundle allocation score on ra2: -INFINITY
+container_color: redis-bundle allocation score on ra3: 0
+container_color: redis-bundle-0 allocation score on ra1: 0
+container_color: redis-bundle-0 allocation score on ra2: 0
+container_color: redis-bundle-0 allocation score on ra3: 0
+container_color: redis-bundle-1 allocation score on ra1: 0
+container_color: redis-bundle-1 allocation score on ra2: 0
+container_color: redis-bundle-1 allocation score on ra3: 0
+container_color: redis-bundle-2 allocation score on ra1: 0
+container_color: redis-bundle-2 allocation score on ra2: 0
+container_color: redis-bundle-2 allocation score on ra3: 0
+container_color: redis-bundle-docker-0 allocation score on ra1: 0
+container_color: redis-bundle-docker-0 allocation score on ra2: -INFINITY
+container_color: redis-bundle-docker-0 allocation score on ra3: 0
+container_color: redis-bundle-docker-1 allocation score on ra1: 0
+container_color: redis-bundle-docker-1 allocation score on ra2: -INFINITY
+container_color: redis-bundle-docker-1 allocation score on ra3: 0
+container_color: redis-bundle-docker-2 allocation score on ra1: 0
+container_color: redis-bundle-docker-2 allocation score on ra2: -INFINITY
+container_color: redis-bundle-docker-2 allocation score on ra3: 0
+container_color: redis-bundle-master allocation score on ra1: 0
+container_color: redis-bundle-master allocation score on ra2: 0
+container_color: redis-bundle-master allocation score on ra3: 0
+container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
+container_color: redis-bundle-master allocation score on redis-bundle-1: -INFINITY
+container_color: redis-bundle-master allocation score on redis-bundle-2: -INFINITY
+container_color: redis:0 allocation score on redis-bundle-0: 501
+container_color: redis:1 allocation score on redis-bundle-1: 500
+container_color: redis:2 allocation score on redis-bundle-2: 501
+native_color: redis-bundle-0 allocation score on ra1: 10000
+native_color: redis-bundle-0 allocation score on ra2: 0
+native_color: redis-bundle-0 allocation score on ra3: 0
+native_color: redis-bundle-1 allocation score on ra1: 0
+native_color: redis-bundle-1 allocation score on ra2: 0
+native_color: redis-bundle-1 allocation score on ra3: 0
+native_color: redis-bundle-2 allocation score on ra1: 0
+native_color: redis-bundle-2 allocation score on ra2: 0
+native_color: redis-bundle-2 allocation score on ra3: 10000
+native_color: redis-bundle-docker-0 allocation score on ra1: 0
+native_color: redis-bundle-docker-0 allocation score on ra2: -INFINITY
+native_color: redis-bundle-docker-0 allocation score on ra3: 0
+native_color: redis-bundle-docker-1 allocation score on ra1: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on ra2: -INFINITY
+native_color: redis-bundle-docker-1 allocation score on ra3: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on ra1: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on ra2: -INFINITY
+native_color: redis-bundle-docker-2 allocation score on ra3: 0
+native_color: redis:0 allocation score on redis-bundle-0: INFINITY
+native_color: redis:1 allocation score on redis-bundle-1: INFINITY
+native_color: redis:2 allocation score on redis-bundle-2: INFINITY
+redis:0 promotion score on redis-bundle-0: 1
+redis:1 promotion score on redis-bundle-1: -1
+redis:2 promotion score on redis-bundle-2: 1
diff --git a/pengine/test10/notify-behind-stopping-remote.summary b/pengine/test10/notify-behind-stopping-remote.summary
new file mode 100644
index 0000000..b9342b9
--- /dev/null
+++ b/pengine/test10/notify-behind-stopping-remote.summary
@@ -0,0 +1,58 @@
+Using the original execution date of: 2018-11-22 20:36:07Z
+
+Current cluster status:
+Online: [ ra1 ra2 ra3 ]
+Containers: [ redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ]
+
+ Docker container set: redis-bundle [docker.io/tripleoqueens/centos-binary-redis:current-tripleo-rdo]
+ redis-bundle-0 (ocf::heartbeat:redis): Slave ra1
+ redis-bundle-1 (ocf::heartbeat:redis): Stopped ra2
+ redis-bundle-2 (ocf::heartbeat:redis): Slave ra3
+
+Transition Summary:
+ * Promote redis:0 ( Slave -> Master redis-bundle-0 )
+ * Stop redis-bundle-docker-1 ( ra2 ) due to node availability
+ * Stop redis-bundle-1 ( ra2 ) due to unrunnable redis-bundle-docker-1 start
+ * Start redis:1 ( redis-bundle-1 ) due to unrunnable redis-bundle-docker-1 start (blocked)
+
+Executing cluster transition:
+ * Resource action: redis cancel=45000 on redis-bundle-0
+ * Resource action: redis cancel=60000 on redis-bundle-0
+ * Pseudo action: redis-bundle-master_pre_notify_start_0
+ * Resource action: redis-bundle-1 stop on ra2
+ * Pseudo action: redis-bundle_stop_0
+ * Pseudo action: redis-bundle-master_confirmed-pre_notify_start_0
+ * Resource action: redis-bundle-docker-1 stop on ra2
+ * Pseudo action: redis-bundle_stopped_0
+ * Pseudo action: redis-bundle_start_0
+ * Pseudo action: all_stopped
+ * Pseudo action: redis-bundle-master_start_0
+ * Pseudo action: redis-bundle-master_running_0
+ * Pseudo action: redis-bundle-master_post_notify_running_0
+ * Pseudo action: redis-bundle-master_confirmed-post_notify_running_0
+ * Pseudo action: redis-bundle_running_0
+ * Pseudo action: redis-bundle-master_pre_notify_promote_0
+ * Pseudo action: redis-bundle_promote_0
+ * Resource action: redis notify on redis-bundle-0
+ * Resource action: redis notify on redis-bundle-2
+ * Pseudo action: redis-bundle-master_confirmed-pre_notify_promote_0
+ * Pseudo action: redis-bundle-master_promote_0
+ * Resource action: redis promote on redis-bundle-0
+ * Pseudo action: redis-bundle-master_promoted_0
+ * Pseudo action: redis-bundle-master_post_notify_promoted_0
+ * Resource action: redis notify on redis-bundle-0
+ * Resource action: redis notify on redis-bundle-2
+ * Pseudo action: redis-bundle-master_confirmed-post_notify_promoted_0
+ * Pseudo action: redis-bundle_promoted_0
+ * Resource action: redis monitor=20000 on redis-bundle-0
+Using the original execution date of: 2018-11-22 20:36:07Z
+
+Revised cluster status:
+Online: [ ra1 ra2 ra3 ]
+Containers: [ redis-bundle-0:redis-bundle-docker-0 redis-bundle-2:redis-bundle-docker-2 ]
+
+ Docker container set: redis-bundle [docker.io/tripleoqueens/centos-binary-redis:current-tripleo-rdo]
+ redis-bundle-0 (ocf::heartbeat:redis): Master ra1
+ redis-bundle-1 (ocf::heartbeat:redis): Stopped
+ redis-bundle-2 (ocf::heartbeat:redis): Slave ra3
+
diff --git a/pengine/test10/notify-behind-stopping-remote.xml b/pengine/test10/notify-behind-stopping-remote.xml
new file mode 100644
index 0000000..66351b8
--- /dev/null
+++ b/pengine/test10/notify-behind-stopping-remote.xml
@@ -0,0 +1,187 @@
+<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.10" epoch="53" num_updates="5" admin_epoch="0" cib-last-written="Thu Nov 22 15:35:52 2018" update-origin="ra2" update-client="crm_resource" update-user="root" have-quorum="1" dc-uuid="3" execution-date="1542918967">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.19-8.el7_6.1-c3c624ea3d"/>
+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="ratester"/>
+ <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1542909574"/>
+ </cluster_property_set>
+ <cluster_property_set id="redis_replication">
+ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="ra2"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="ra1"/>
+ <node id="2" uname="ra2"/>
+ <node id="3" uname="ra3"/>
+ </nodes>
+ <resources>
+ <bundle id="redis-bundle">
+ <docker masters="1" network="host" replicas="3" image="docker.io/tripleoqueens/centos-binary-redis:current-tripleo-rdo" run-command="/usr/sbin/pacemaker_remoted" options="--user=root --log-driver=journald"/>
+ <network control-port="3123"/>
+ <storage>
+ <storage-mapping target-dir="/dev/log" id="map0" source-dir="/dev/log"/>
+ <storage-mapping target-dir="/etc/libqb/force-filesystem-sockets" options="ro" id="map1" source-dir="/dev/zero"/>
+ <storage-mapping target-dir="/etc/hosts" options="ro" id="map2" source-dir="/etc/hosts"/>
+ <storage-mapping target-dir="/etc/localtime" options="ro" id="map3" source-dir="/etc/localtime"/>
+ <storage-mapping target-dir="/etc/redis.conf" options="ro" id="map4" source-dir="/etc/redis.conf"/>
+ <storage-mapping target-dir="/var/lib/redis" options="rw" id="map5" source-dir="/var/lib/redis"/>
+ <storage-mapping target-dir="/var/log/redis" options="rw" id="map6" source-dir="/var/log/redis"/>
+ <storage-mapping target-dir="/var/run/redis" options="rw" id="map7" source-dir="/var/run/redis"/>
+ <storage-mapping target-dir="/usr/lib/ocf" options="rw" id="map8" source-dir="/usr/lib/ocf"/>
+ <storage-mapping target-dir="/var/log/pacemaker" options="rw" id="pcmk1" source-dir="/var/log/pacemaker"/>
+ </storage>
+ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
+ <instance_attributes id="redis-instance_attributes">
+ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
+ </instance_attributes>
+ <meta_attributes id="redis-meta_attributes">
+ <nvpair id="redis-meta_attributes-container-attribute-target" name="container-attribute-target" value="host"/>
+ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
+ </meta_attributes>
+ <operations>
+ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120s"/>
+ <op id="redis-monitor-interval-45s" interval="45s" name="monitor" timeout="60s"/>
+ <op id="redis-monitor-interval-20s" interval="20s" name="monitor" role="Master" timeout="60s"/>
+ <op id="redis-monitor-interval-60s" interval="60s" name="monitor" role="Slave" timeout="60s"/>
+ <op id="redis-notify-interval-0s" interval="0s" name="notify" timeout="90s"/>
+ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120s"/>
+ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="120s"/>
+ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
+ </operations>
+ </primitive>
+ </bundle>
+ </resources>
+ <constraints>
+ <rsc_location id="cli-ban-redis-bundle-on-ra2" rsc="redis-bundle" role="Started" node="ra2" score="-INFINITY"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state id="2" uname="ra2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="2">
+ <lrm_resources>
+ <lrm_resource id="redis-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:38:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;7:38:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="53" rc-code="7" op-status="0" interval="0" last-run="1542907811" last-rc-change="1542907811" exec-time="65" queue-time="0" op-digest="317386a3af010667314d88ddf93a0150"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;7:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1542907815" last-rc-change="1542907815" exec-time="122" queue-time="0" op-digest="cabda3f80e7220806c7022b88efab9d4"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="6:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;6:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="81" rc-code="0" op-status="0" interval="0" last-run="1542918685" last-rc-change="1542918685" exec-time="1479" queue-time="0" op-digest="ad90cb2f61c71c84e840c796e44d628a"/>
+ <lrm_rsc_op id="redis-bundle-docker-1_monitor_60000" operation_key="redis-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;7:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="82" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918686" exec-time="173" queue-time="0" op-digest="51d9043cab411f1c983d9539d2997c26"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-0" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-0">
+ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="5:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;5:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="109c7a28bbb65c3c294a591f03377dd3" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-1" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-1">
+ <lrm_rsc_op id="redis-bundle-1_last_0" operation_key="redis-bundle-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="8:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;8:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1542918686" last-rc-change="1542918686" exec-time="0" queue-time="0" op-digest="9c100f1c3496f07f7a953f9cc8fbd746" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="redis-bundle-1_monitor_60000" operation_key="redis-bundle-1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="17:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;17:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="14" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918687" exec-time="0" queue-time="0" op-digest="21306bcef84a89e55ae729d77ceca1dc"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-2" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-2">
+ <lrm_rsc_op id="redis-bundle-2_last_0" operation_key="redis-bundle-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;7:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="6ed7cb9eb2c67afe77784971250638a7" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="2">
+ <instance_attributes id="status-2"/>
+ </transient_attributes>
+ </node_state>
+ <node_state id="1" uname="ra1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="redis-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="4:37:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;4:37:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="57" rc-code="7" op-status="0" interval="0" last-run="1542907811" last-rc-change="1542907811" exec-time="63" queue-time="0" op-digest="0160bbff6f3f012b38edc9bddb631132"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="2:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;2:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="84" rc-code="0" op-status="0" interval="0" last-run="1542918685" last-rc-change="1542918685" exec-time="1504" queue-time="0" op-digest="cabda3f80e7220806c7022b88efab9d4"/>
+ <lrm_rsc_op id="redis-bundle-docker-0_monitor_60000" operation_key="redis-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="3:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;3:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="85" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918686" exec-time="167" queue-time="0" op-digest="922138cf0d1c7127c35f2e174c4af461"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="4:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;4:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="68" rc-code="7" op-status="0" interval="0" last-run="1542907815" last-rc-change="1542907815" exec-time="132" queue-time="0" op-digest="ad90cb2f61c71c84e840c796e44d628a"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-0" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-0">
+ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="4:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;4:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1542918686" last-rc-change="1542918686" exec-time="0" queue-time="0" op-digest="109c7a28bbb65c3c294a591f03377dd3" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="redis-bundle-0_monitor_60000" operation_key="redis-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="12:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;12:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="14" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918687" exec-time="0" queue-time="0" op-digest="478c16eefb4fa513134f527efb7e063a"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-1" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-1">
+ <lrm_rsc_op id="redis-bundle-1_last_0" operation_key="redis-bundle-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="3:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;3:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="9c100f1c3496f07f7a953f9cc8fbd746" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-2" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-2">
+ <lrm_rsc_op id="redis-bundle-2_last_0" operation_key="redis-bundle-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="4:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;4:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="6ed7cb9eb2c67afe77784971250638a7" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-master-redis" name="master-redis" value="1"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state id="3" uname="ra3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="redis-bundle-docker-2" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="10:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;10:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="77" rc-code="0" op-status="0" interval="0" last-run="1542918685" last-rc-change="1542918685" exec-time="1501" queue-time="0" op-digest="317386a3af010667314d88ddf93a0150"/>
+ <lrm_rsc_op id="redis-bundle-docker-2_monitor_60000" operation_key="redis-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="11:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;11:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="78" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918687" exec-time="165" queue-time="0" op-digest="b4fcc691b4f687c09bb3dd0610b658ca"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="12:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;12:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="57" rc-code="7" op-status="0" interval="0" last-run="1542907815" last-rc-change="1542907815" exec-time="129" queue-time="0" op-digest="cabda3f80e7220806c7022b88efab9d4"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-docker-1" type="docker" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="14:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;14:39:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1542907815" last-rc-change="1542907815" exec-time="111" queue-time="0" op-digest="ad90cb2f61c71c84e840c796e44d628a"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-0" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-0">
+ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="8:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;8:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="109c7a28bbb65c3c294a591f03377dd3" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-1" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-1">
+ <lrm_rsc_op id="redis-bundle-1_last_0" operation_key="redis-bundle-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="9:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:7;9:40:7:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1542907819" last-rc-change="1542907819" exec-time="0" queue-time="0" op-digest="9c100f1c3496f07f7a953f9cc8fbd746" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="redis-bundle-2" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-2">
+ <lrm_rsc_op id="redis-bundle-2_last_0" operation_key="redis-bundle-2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="12:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;12:142:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1542918687" last-rc-change="1542918687" exec-time="0" queue-time="0" op-digest="6ed7cb9eb2c67afe77784971250638a7" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="redis-bundle-2_monitor_60000" operation_key="redis-bundle-2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="22:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;22:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="14" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918688" exec-time="0" queue-time="0" op-digest="6815f4f1d3f361f086d1e45371d9cb53"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-master-redis" name="master-redis" value="1"/>
+ </instance_attributes>
+ </transient_attributes>
+ </node_state>
+ <node_state remote_node="true" id="redis-bundle-0" uname="redis-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
+ <lrm id="redis-bundle-0">
+ <lrm_resources>
+ <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis_last_0" operation_key="redis_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="27:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;27:143:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="8" rc-code="0" op-status="0" interval="0" last-run="1542918688" last-rc-change="1542918688" exec-time="5253" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
+ <lrm_rsc_op id="redis_monitor_60000" operation_key="redis_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="32:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;32:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="78" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918713" exec-time="1195" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
+ <lrm_rsc_op id="redis_monitor_45000" operation_key="redis_monitor_45000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="31:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;31:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra1" call-id="79" rc-code="0" op-status="0" interval="45000" last-rc-change="1542918715" exec-time="1191" queue-time="1114" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state remote_node="true" id="redis-bundle-1" uname="redis-bundle-1" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
+ <lrm id="redis-bundle-1">
+ <lrm_resources>
+ <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis_last_0" operation_key="redis_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="30:148:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;30:148:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="143" rc-code="0" op-status="0" interval="0" last-run="1542918963" last-rc-change="1542918963" exec-time="2398" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
+ <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="28:145:8:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:8;28:145:8:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra2" call-id="59" rc-code="8" op-status="0" interval="20000" last-rc-change="1542918715" exec-time="1210" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state remote_node="true" id="redis-bundle-2" uname="redis-bundle-2" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
+ <lrm id="redis-bundle-2">
+ <lrm_resources>
+ <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="redis_last_0" operation_key="redis_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="32:144:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;32:144:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="8" rc-code="0" op-status="0" interval="0" last-run="1542918700" last-rc-change="1542918700" exec-time="5173" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
+ <lrm_rsc_op id="redis_monitor_60000" operation_key="redis_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="36:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;36:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="77" rc-code="0" op-status="0" interval="60000" last-rc-change="1542918714" exec-time="1184" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
+ <lrm_rsc_op id="redis_monitor_45000" operation_key="redis_monitor_45000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="35:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" transition-magic="0:0;35:145:0:ae78ea4e-f87a-4433-b26e-94016870d3fc" exit-reason="" on_node="ra3" call-id="78" rc-code="0" op-status="0" interval="45000" last-rc-change="1542918715" exec-time="1157" queue-time="1102" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ </status>
+</cib>
--
1.8.3.1
From 90bed507285d23218617f0cd520d788ba246761e Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Wed, 28 Nov 2018 18:30:50 -0600
Subject: [PATCH 3/5] Test: scheduler: update bundle-order-fencing test for
notification change
Don't schedule clone notifications on a Pacemaker Remote node that
has just been fenced.
---
pengine/test10/bundle-order-fencing.dot | 8 -
pengine/test10/bundle-order-fencing.exp | 246 ++++++++++++----------------
pengine/test10/bundle-order-fencing.summary | 2 -
3 files changed, 107 insertions(+), 149 deletions(-)
diff --git a/pengine/test10/bundle-order-fencing.dot b/pengine/test10/bundle-order-fencing.dot
index e53a062..1e2721b 100644
--- a/pengine/test10/bundle-order-fencing.dot
+++ b/pengine/test10/bundle-order-fencing.dot
@@ -210,8 +210,6 @@ digraph "g" {
"redis-bundle-master_confirmed-post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"]
"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis-bundle_promoted_0" [ style = bold]
"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold]
-"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed]
-"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed]
"redis-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]
"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle-master_pre_notify_promote_0" [ style = bold]
"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle_running_0" [ style = bold]
@@ -247,7 +245,6 @@ digraph "g" {
"redis-bundle-master_post_notify_demoted_0" -> "redis_post_notify_demoted_0 redis-bundle-2" [ style = bold]
"redis-bundle-master_post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"]
"redis-bundle-master_post_notify_promoted_0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
-"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-0" [ style = bold]
"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-1" [ style = bold]
"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-2" [ style = bold]
"redis-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]
@@ -262,7 +259,6 @@ digraph "g" {
"redis-bundle-master_pre_notify_demote_0" -> "redis_pre_notify_demote_0 redis-bundle-2" [ style = bold]
"redis-bundle-master_pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"]
"redis-bundle-master_pre_notify_promote_0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
-"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-0" [ style = bold]
"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-1" [ style = bold]
"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-2" [ style = bold]
"redis-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"]
@@ -325,8 +321,6 @@ digraph "g" {
"redis_post_notify_demoted_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_demoted_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold]
"redis_post_notify_demoted_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
-"redis_post_notify_promoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
-"redis_post_notify_promoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_promoted_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
"redis_post_notify_promoted_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_promoted_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold]
@@ -345,8 +339,6 @@ digraph "g" {
"redis_pre_notify_demote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
"redis_pre_notify_demote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold]
"redis_pre_notify_demote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
-"redis_pre_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
-"redis_pre_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
"redis_pre_notify_promote_0 redis-bundle-1" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
"redis_pre_notify_promote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
"redis_pre_notify_promote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold]
diff --git a/pengine/test10/bundle-order-fencing.exp b/pengine/test10/bundle-order-fencing.exp
index 2b8f5cf..84bffaa 100644
--- a/pengine/test10/bundle-order-fencing.exp
+++ b/pengine/test10/bundle-order-fencing.exp
@@ -521,32 +521,6 @@
</synapse>
<synapse id="39" priority="1000000">
<action_set>
- <rsc_op id="240" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1">
- <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
- <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="20000" wait_last_known_master="true"/>
- </rsc_op>
- </action_set>
- <inputs>
- <trigger>
- <pseudo_event id="161" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
- </trigger>
- </inputs>
- </synapse>
- <synapse id="40">
- <action_set>
- <rsc_op id="239" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1">
- <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
- <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="20000" wait_last_known_master="true"/>
- </rsc_op>
- </action_set>
- <inputs>
- <trigger>
- <pseudo_event id="159" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
- </trigger>
- </inputs>
- </synapse>
- <synapse id="41" priority="1000000">
- <action_set>
<pseudo_event id="216" operation="notified" operation_key="redis_notified_0" internal_operation_key="redis:0_confirmed-post_notify_stonith_0">
<attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stonith" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" wait_last_known_master="true"/>
</pseudo_event>
@@ -563,7 +537,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="42" priority="1000000">
+ <synapse id="40" priority="1000000">
<action_set>
<pseudo_event id="215" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:0_post_notify_stonith_0">
<attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stonith" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" wait_last_known_master="true"/>
@@ -575,7 +549,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="43">
+ <synapse id="41">
<action_set>
<pseudo_event id="141" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0">
<attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_timeout="200000" wait_last_known_master="true"/>
@@ -593,7 +567,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="44">
+ <synapse id="42">
<action_set>
<pseudo_event id="140" operation="demote" operation_key="redis_demote_0" internal_operation_key="redis:0_demote_0">
<attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_physical_host="controller-0" CRM_meta_timeout="120000" wait_last_known_master="true"/>
@@ -605,9 +579,9 @@
</trigger>
</inputs>
</synapse>
- <synapse id="45" priority="1000000">
+ <synapse id="43" priority="1000000">
<action_set>
- <rsc_op id="246" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <rsc_op id="244" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
<primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
<attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
</rsc_op>
@@ -618,9 +592,9 @@
</trigger>
</inputs>
</synapse>
- <synapse id="46">
+ <synapse id="44">
<action_set>
- <rsc_op id="245" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <rsc_op id="243" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
<primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
<attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
</rsc_op>
@@ -631,9 +605,9 @@
</trigger>
</inputs>
</synapse>
- <synapse id="47" priority="1000000">
+ <synapse id="45" priority="1000000">
<action_set>
- <rsc_op id="242" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <rsc_op id="240" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
<primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
<attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
</rsc_op>
@@ -644,9 +618,9 @@
</trigger>
</inputs>
</synapse>
- <synapse id="48">
+ <synapse id="46">
<action_set>
- <rsc_op id="241" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
+ <rsc_op id="239" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
<primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
<attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_physical_host="controller-1" CRM_meta_timeout="20000" wait_last_known_master="true"/>
</rsc_op>
@@ -657,7 +631,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="49">
+ <synapse id="47">
<action_set>
<rsc_op id="237" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:1_pre_notify_stop_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
<primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
@@ -670,7 +644,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="50" priority="1000000">
+ <synapse id="48" priority="1000000">
<action_set>
<rsc_op id="217" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:1_post_notify_stop_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
<primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
@@ -686,7 +660,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="51">
+ <synapse id="49">
<action_set>
<rsc_op id="137" operation="monitor" operation_key="redis_monitor_20000" internal_operation_key="redis:1_monitor_20000" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
<primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
@@ -711,7 +685,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="52">
+ <synapse id="50">
<action_set>
<rsc_op id="136" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:1_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
<primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
@@ -733,7 +707,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="53">
+ <synapse id="51">
<action_set>
<rsc_op id="28" operation="cancel" operation_key="redis_monitor_45000" internal_operation_key="redis:1_monitor_45000" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
<primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
@@ -742,7 +716,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="54">
+ <synapse id="52">
<action_set>
<rsc_op id="27" operation="cancel" operation_key="redis_monitor_60000" internal_operation_key="redis:1_monitor_60000" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1">
<primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
@@ -751,9 +725,9 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="55" priority="1000000">
+ <synapse id="53" priority="1000000">
<action_set>
- <rsc_op id="248" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <rsc_op id="246" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
<primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
<attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
</rsc_op>
@@ -764,9 +738,9 @@
</trigger>
</inputs>
</synapse>
- <synapse id="56">
+ <synapse id="54">
<action_set>
- <rsc_op id="247" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <rsc_op id="245" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
<primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
<attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="demote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
</rsc_op>
@@ -777,9 +751,9 @@
</trigger>
</inputs>
</synapse>
- <synapse id="57" priority="1000000">
+ <synapse id="55" priority="1000000">
<action_set>
- <rsc_op id="244" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <rsc_op id="242" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
<primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
<attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="post" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
</rsc_op>
@@ -790,9 +764,9 @@
</trigger>
</inputs>
</synapse>
- <synapse id="58">
+ <synapse id="56">
<action_set>
- <rsc_op id="243" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
+ <rsc_op id="241" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
<primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
<attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_container_attribute_target="host" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_all_hosts="controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2 controller-0 controller-1 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-bundle-0 galera-bundle-1 galera-bundle-2 rabbitmq-bundle-0 rabbitmq-bundle-1 rabbitmq-bundle-2 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="redis-bundle-1 redis-bundle-0 redis-bundle-2 controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource="redis:0" CRM_meta_notify_demote_uname="redis-bundle-0" CRM_meta_notify_inactive_resource=" " CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:0" CRM_meta_notify_master_uname="redis-bundle-0" CRM_meta_notify_operation="promote" CRM_meta_notify_promote_resource="redis:1" CRM_meta_notify_promote_uname="redis-bundle-1" CRM_meta_notify_slave_resource="redis:1 redis:2" CRM_meta_notify_slave_uname="redis-bundle-1 redis-bundle-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="redis-bundle-0" CRM_meta_notify_type="pre" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_physical_host="controller-2" CRM_meta_timeout="20000" wait_last_known_master="true"/>
</rsc_op>
@@ -803,7 +777,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="59">
+ <synapse id="57">
<action_set>
<rsc_op id="238" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:2_pre_notify_stop_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
<primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
@@ -816,7 +790,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="60" priority="1000000">
+ <synapse id="58" priority="1000000">
<action_set>
<rsc_op id="218" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:2_post_notify_stop_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2">
<primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
@@ -832,7 +806,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="61" priority="1000000">
+ <synapse id="59" priority="1000000">
<action_set>
<pseudo_event id="168" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_demoted_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="demote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
@@ -843,14 +817,14 @@
<pseudo_event id="167" operation="notify" operation_key="redis-bundle-master_post_notify_demoted_0"/>
</trigger>
<trigger>
- <rsc_op id="246" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ <rsc_op id="244" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:1_post_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
</trigger>
<trigger>
- <rsc_op id="248" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ <rsc_op id="246" operation="notify" operation_key="redis_post_notify_demote_0" internal_operation_key="redis:2_post_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
</trigger>
</inputs>
</synapse>
- <synapse id="62" priority="1000000">
+ <synapse id="60" priority="1000000">
<action_set>
<pseudo_event id="167" operation="notify" operation_key="redis-bundle-master_post_notify_demoted_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="demoted" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="demote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
@@ -865,7 +839,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="63">
+ <synapse id="61">
<action_set>
<pseudo_event id="166" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_demote_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="demote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
@@ -876,14 +850,14 @@
<pseudo_event id="165" operation="notify" operation_key="redis-bundle-master_pre_notify_demote_0"/>
</trigger>
<trigger>
- <rsc_op id="245" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ <rsc_op id="243" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:1_pre_notify_demote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
</trigger>
<trigger>
- <rsc_op id="247" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ <rsc_op id="245" operation="notify" operation_key="redis_pre_notify_demote_0" internal_operation_key="redis:2_pre_notify_demote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
</trigger>
</inputs>
</synapse>
- <synapse id="64">
+ <synapse id="62">
<action_set>
<pseudo_event id="165" operation="notify" operation_key="redis-bundle-master_pre_notify_demote_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="demote" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="demote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
@@ -891,7 +865,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="65" priority="1000000">
+ <synapse id="63" priority="1000000">
<action_set>
<pseudo_event id="164" operation="demoted" operation_key="redis-bundle-master_demoted_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
@@ -906,7 +880,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="66">
+ <synapse id="64">
<action_set>
<pseudo_event id="163" operation="demote" operation_key="redis-bundle-master_demote_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
@@ -921,7 +895,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="67" priority="1000000">
+ <synapse id="65" priority="1000000">
<action_set>
<pseudo_event id="162" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_promoted_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="promote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
@@ -932,17 +906,14 @@
<pseudo_event id="161" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0"/>
</trigger>
<trigger>
- <rsc_op id="240" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:0_post_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1"/>
+ <rsc_op id="240" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
</trigger>
<trigger>
- <rsc_op id="242" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:1_post_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
- </trigger>
- <trigger>
- <rsc_op id="244" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ <rsc_op id="242" operation="notify" operation_key="redis_post_notify_promote_0" internal_operation_key="redis:2_post_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
</trigger>
</inputs>
</synapse>
- <synapse id="68" priority="1000000">
+ <synapse id="66" priority="1000000">
<action_set>
<pseudo_event id="161" operation="notify" operation_key="redis-bundle-master_post_notify_promoted_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promoted" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="promote" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
@@ -957,7 +928,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="69">
+ <synapse id="67">
<action_set>
<pseudo_event id="160" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_promote_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="promote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
@@ -968,17 +939,14 @@
<pseudo_event id="159" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0"/>
</trigger>
<trigger>
- <rsc_op id="239" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:0_pre_notify_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="controller-1"/>
- </trigger>
- <trigger>
- <rsc_op id="241" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
+ <rsc_op id="239" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:1_pre_notify_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="controller-1"/>
</trigger>
<trigger>
- <rsc_op id="243" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
+ <rsc_op id="241" operation="notify" operation_key="redis_pre_notify_promote_0" internal_operation_key="redis:2_pre_notify_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="controller-2"/>
</trigger>
</inputs>
</synapse>
- <synapse id="70">
+ <synapse id="68">
<action_set>
<pseudo_event id="159" operation="notify" operation_key="redis-bundle-master_pre_notify_promote_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="promote" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="promote" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
@@ -996,7 +964,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="71" priority="1000000">
+ <synapse id="69" priority="1000000">
<action_set>
<pseudo_event id="158" operation="promoted" operation_key="redis-bundle-master_promoted_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
@@ -1008,7 +976,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="72">
+ <synapse id="70">
<action_set>
<pseudo_event id="157" operation="promote" operation_key="redis-bundle-master_promote_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
@@ -1032,7 +1000,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="73" priority="1000000">
+ <synapse id="71" priority="1000000">
<action_set>
<pseudo_event id="156" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_stopped_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
@@ -1050,7 +1018,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="74" priority="1000000">
+ <synapse id="72" priority="1000000">
<action_set>
<pseudo_event id="155" operation="notify" operation_key="redis-bundle-master_post_notify_stopped_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
@@ -1065,7 +1033,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="75">
+ <synapse id="73">
<action_set>
<pseudo_event id="154" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_stop_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
@@ -1083,7 +1051,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="76">
+ <synapse id="74">
<action_set>
<pseudo_event id="153" operation="notify" operation_key="redis-bundle-master_pre_notify_stop_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
@@ -1095,7 +1063,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="77" priority="1000000">
+ <synapse id="75" priority="1000000">
<action_set>
<pseudo_event id="152" operation="stopped" operation_key="redis-bundle-master_stopped_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
@@ -1110,7 +1078,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="78">
+ <synapse id="76">
<action_set>
<pseudo_event id="151" operation="stop" operation_key="redis-bundle-master_stop_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
@@ -1131,7 +1099,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="79" priority="1000000">
+ <synapse id="77" priority="1000000">
<action_set>
<pseudo_event id="150" operation="notified" operation_key="redis-bundle-master_confirmed-post_notify_running_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
@@ -1143,7 +1111,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="80" priority="1000000">
+ <synapse id="78" priority="1000000">
<action_set>
<pseudo_event id="149" operation="notify" operation_key="redis-bundle-master_post_notify_running_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
@@ -1158,7 +1126,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="81">
+ <synapse id="79">
<action_set>
<pseudo_event id="148" operation="notified" operation_key="redis-bundle-master_confirmed-pre_notify_start_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
@@ -1170,7 +1138,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="82">
+ <synapse id="80">
<action_set>
<pseudo_event id="147" operation="notify" operation_key="redis-bundle-master_pre_notify_start_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
@@ -1185,7 +1153,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="83" priority="1000000">
+ <synapse id="81" priority="1000000">
<action_set>
<pseudo_event id="146" operation="running" operation_key="redis-bundle-master_running_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
@@ -1197,7 +1165,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="84">
+ <synapse id="82">
<action_set>
<pseudo_event id="145" operation="start" operation_key="redis-bundle-master_start_0">
<attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
@@ -1218,7 +1186,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="85">
+ <synapse id="83">
<action_set>
<pseudo_event id="118" operation="stop" operation_key="redis-bundle-docker-0_stop_0">
<attributes CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/rhosp12/openstack-redis-docker:pcmklatest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/redis/:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
@@ -1236,7 +1204,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="86">
+ <synapse id="84">
<action_set>
<pseudo_event id="119" operation="stop" operation_key="redis-bundle-0_stop_0">
<attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_timeout="20000" addr="controller-0" port="3124"/>
@@ -1244,7 +1212,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="87">
+ <synapse id="85">
<action_set>
<rsc_op id="40" operation="monitor" operation_key="redis-bundle-0_monitor_0" on_node="controller-2" on_node_uuid="3">
<primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
@@ -1253,7 +1221,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="88">
+ <synapse id="86">
<action_set>
<rsc_op id="34" operation="monitor" operation_key="redis-bundle-0_monitor_0" on_node="controller-1" on_node_uuid="2">
<primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
@@ -1262,7 +1230,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="89">
+ <synapse id="87">
<action_set>
<rsc_op id="41" operation="monitor" operation_key="redis-bundle-1_monitor_0" on_node="controller-2" on_node_uuid="3">
<primitive id="redis-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
@@ -1271,7 +1239,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="90">
+ <synapse id="88">
<action_set>
<rsc_op id="35" operation="monitor" operation_key="redis-bundle-2_monitor_0" on_node="controller-1" on_node_uuid="2">
<primitive id="redis-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
@@ -1280,7 +1248,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="91">
+ <synapse id="89">
<action_set>
<rsc_op id="175" operation="monitor" operation_key="ip-192.168.24.7_monitor_10000" on_node="controller-2" on_node_uuid="3">
<primitive id="ip-192.168.24.7" class="ocf" provider="heartbeat" type="IPaddr2"/>
@@ -1293,7 +1261,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="92">
+ <synapse id="90">
<action_set>
<rsc_op id="174" operation="start" operation_key="ip-192.168.24.7_start_0" on_node="controller-2" on_node_uuid="3">
<primitive id="ip-192.168.24.7" class="ocf" provider="heartbeat" type="IPaddr2"/>
@@ -1309,7 +1277,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="93">
+ <synapse id="91">
<action_set>
<pseudo_event id="173" operation="stop" operation_key="ip-192.168.24.7_stop_0">
<attributes CRM_meta_name="stop" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.24.7"/>
@@ -1324,7 +1292,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="94">
+ <synapse id="92">
<action_set>
<rsc_op id="178" operation="monitor" operation_key="ip-10.0.0.109_monitor_10000" on_node="controller-1" on_node_uuid="2">
<primitive id="ip-10.0.0.109" class="ocf" provider="heartbeat" type="IPaddr2"/>
@@ -1337,7 +1305,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="95">
+ <synapse id="93">
<action_set>
<rsc_op id="177" operation="start" operation_key="ip-10.0.0.109_start_0" on_node="controller-1" on_node_uuid="2">
<primitive id="ip-10.0.0.109" class="ocf" provider="heartbeat" type="IPaddr2"/>
@@ -1353,7 +1321,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="96">
+ <synapse id="94">
<action_set>
<pseudo_event id="176" operation="stop" operation_key="ip-10.0.0.109_stop_0">
<attributes CRM_meta_name="stop" CRM_meta_timeout="20000" cidr_netmask="32" ip="10.0.0.109"/>
@@ -1368,7 +1336,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="97">
+ <synapse id="95">
<action_set>
<rsc_op id="187" operation="monitor" operation_key="ip-172.17.4.11_monitor_10000" on_node="controller-1" on_node_uuid="2">
<primitive id="ip-172.17.4.11" class="ocf" provider="heartbeat" type="IPaddr2"/>
@@ -1381,7 +1349,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="98">
+ <synapse id="96">
<action_set>
<rsc_op id="186" operation="start" operation_key="ip-172.17.4.11_start_0" on_node="controller-1" on_node_uuid="2">
<primitive id="ip-172.17.4.11" class="ocf" provider="heartbeat" type="IPaddr2"/>
@@ -1397,7 +1365,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="99">
+ <synapse id="97">
<action_set>
<pseudo_event id="185" operation="stop" operation_key="ip-172.17.4.11_stop_0">
<attributes CRM_meta_name="stop" CRM_meta_timeout="20000" cidr_netmask="32" ip="172.17.4.11"/>
@@ -1412,7 +1380,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="100">
+ <synapse id="98">
<action_set>
<pseudo_event id="188" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0">
<attributes CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/rhosp12/openstack-haproxy-docker:pcmklatest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no -e PCMK_stderr=1 --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/puppet-generated/haproxy/:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
@@ -1427,7 +1395,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="101">
+ <synapse id="99">
<action_set>
<rsc_op id="203" operation="monitor" operation_key="stonith-fence_ipmilan-5254003e8e97_monitor_60000" on_node="controller-1" on_node_uuid="2">
<primitive id="stonith-fence_ipmilan-5254003e8e97" class="stonith" type="fence_ipmilan"/>
@@ -1440,7 +1408,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="102">
+ <synapse id="100">
<action_set>
<rsc_op id="202" operation="start" operation_key="stonith-fence_ipmilan-5254003e8e97_start_0" on_node="controller-1" on_node_uuid="2">
<primitive id="stonith-fence_ipmilan-5254003e8e97" class="stonith" type="fence_ipmilan"/>
@@ -1453,7 +1421,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="103">
+ <synapse id="101">
<action_set>
<pseudo_event id="201" operation="stop" operation_key="stonith-fence_ipmilan-5254003e8e97_stop_0">
<attributes CRM_meta_timeout="20000" action="reboot" ipaddr="172.16.0.1" ipport="6231" lanplus="true" login="admin" passwd="password" pcmk_host_list="controller-2" privlvl="administrator"/>
@@ -1461,7 +1429,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="104">
+ <synapse id="102">
<action_set>
<rsc_op id="206" operation="monitor" operation_key="stonith-fence_ipmilan-5254000dcb3f_monitor_60000" on_node="controller-2" on_node_uuid="3">
<primitive id="stonith-fence_ipmilan-5254000dcb3f" class="stonith" type="fence_ipmilan"/>
@@ -1474,7 +1442,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="105">
+ <synapse id="103">
<action_set>
<rsc_op id="205" operation="start" operation_key="stonith-fence_ipmilan-5254000dcb3f_start_0" on_node="controller-2" on_node_uuid="3">
<primitive id="stonith-fence_ipmilan-5254000dcb3f" class="stonith" type="fence_ipmilan"/>
@@ -1487,7 +1455,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="106">
+ <synapse id="104">
<action_set>
<pseudo_event id="204" operation="stop" operation_key="stonith-fence_ipmilan-5254000dcb3f_stop_0">
<attributes CRM_meta_timeout="20000" action="reboot" ipaddr="172.16.0.1" ipport="6230" lanplus="true" login="admin" passwd="password" pcmk_host_list="controller-1" privlvl="administrator"/>
@@ -1495,7 +1463,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="107">
+ <synapse id="105">
<action_set>
<pseudo_event id="214" operation="stonith" operation_key="stonith-redis-bundle-0-off" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0">
<attributes CRM_meta_master_redis="1" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_stonith_action="off" />
@@ -1510,7 +1478,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="108">
+ <synapse id="106">
<action_set>
<pseudo_event id="209" operation="stonith" operation_key="stonith-rabbitmq-bundle-0-off" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0">
<attributes CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_rmq_node_attr_last_known_rabbitmq="rabbit@controller-0" CRM_meta_stonith_action="off" />
@@ -1525,7 +1493,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="109">
+ <synapse id="107">
<action_set>
<pseudo_event id="208" operation="stonith" operation_key="stonith-galera-bundle-0-off" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0">
<attributes CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_stonith_action="off" />
@@ -1540,7 +1508,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="110">
+ <synapse id="108">
<action_set>
<pseudo_event id="207" operation="stonith_complete" operation_key="stonith_complete">
<attributes />
@@ -1561,7 +1529,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="111" priority="1000000">
+ <synapse id="109" priority="1000000">
<action_set>
<pseudo_event id="196" operation="stopped" operation_key="haproxy-bundle_stopped_0">
<attributes CRM_meta_timeout="20000" />
@@ -1573,7 +1541,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="112">
+ <synapse id="110">
<action_set>
<pseudo_event id="195" operation="stop" operation_key="haproxy-bundle_stop_0">
<attributes CRM_meta_timeout="20000" />
@@ -1581,7 +1549,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="113" priority="1000000">
+ <synapse id="111" priority="1000000">
<action_set>
<pseudo_event id="172" operation="demoted" operation_key="redis-bundle_demoted_0">
<attributes CRM_meta_timeout="20000" />
@@ -1596,7 +1564,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="114">
+ <synapse id="112">
<action_set>
<pseudo_event id="171" operation="demote" operation_key="redis-bundle_demote_0">
<attributes CRM_meta_timeout="20000" />
@@ -1604,7 +1572,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="115" priority="1000000">
+ <synapse id="113" priority="1000000">
<action_set>
<pseudo_event id="170" operation="promoted" operation_key="redis-bundle_promoted_0">
<attributes CRM_meta_timeout="20000" />
@@ -1616,7 +1584,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="116">
+ <synapse id="114">
<action_set>
<pseudo_event id="169" operation="promote" operation_key="redis-bundle_promote_0">
<attributes CRM_meta_timeout="20000" />
@@ -1634,7 +1602,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="117" priority="1000000">
+ <synapse id="115" priority="1000000">
<action_set>
<pseudo_event id="133" operation="stopped" operation_key="redis-bundle_stopped_0">
<attributes CRM_meta_timeout="20000" />
@@ -1649,7 +1617,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="118">
+ <synapse id="116">
<action_set>
<pseudo_event id="132" operation="stop" operation_key="redis-bundle_stop_0">
<attributes CRM_meta_timeout="20000" />
@@ -1661,7 +1629,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="119" priority="1000000">
+ <synapse id="117" priority="1000000">
<action_set>
<pseudo_event id="131" operation="running" operation_key="redis-bundle_running_0">
<attributes CRM_meta_timeout="20000" />
@@ -1673,7 +1641,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="120">
+ <synapse id="118">
<action_set>
<pseudo_event id="130" operation="start" operation_key="redis-bundle_start_0">
<attributes CRM_meta_timeout="20000" />
@@ -1688,7 +1656,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="121" priority="1000000">
+ <synapse id="119" priority="1000000">
<action_set>
<pseudo_event id="117" operation="demoted" operation_key="galera-bundle_demoted_0">
<attributes CRM_meta_timeout="20000" />
@@ -1703,7 +1671,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="122">
+ <synapse id="120">
<action_set>
<pseudo_event id="116" operation="demote" operation_key="galera-bundle_demote_0">
<attributes CRM_meta_timeout="20000" />
@@ -1711,7 +1679,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="123" priority="1000000">
+ <synapse id="121" priority="1000000">
<action_set>
<pseudo_event id="92" operation="stopped" operation_key="galera-bundle_stopped_0">
<attributes CRM_meta_timeout="20000" />
@@ -1726,7 +1694,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="124">
+ <synapse id="122">
<action_set>
<pseudo_event id="91" operation="stop" operation_key="galera-bundle_stop_0">
<attributes CRM_meta_timeout="20000" />
@@ -1738,7 +1706,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="125" priority="1000000">
+ <synapse id="123" priority="1000000">
<action_set>
<pseudo_event id="90" operation="running" operation_key="galera-bundle_running_0">
<attributes CRM_meta_timeout="20000" />
@@ -1750,7 +1718,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="126">
+ <synapse id="124">
<action_set>
<pseudo_event id="89" operation="start" operation_key="galera-bundle_start_0">
<attributes CRM_meta_timeout="20000" />
@@ -1765,7 +1733,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="127" priority="1000000">
+ <synapse id="125" priority="1000000">
<action_set>
<pseudo_event id="57" operation="stopped" operation_key="rabbitmq-bundle_stopped_0">
<attributes CRM_meta_timeout="20000" />
@@ -1780,7 +1748,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="128">
+ <synapse id="126">
<action_set>
<pseudo_event id="56" operation="stop" operation_key="rabbitmq-bundle_stop_0">
<attributes CRM_meta_timeout="20000" />
@@ -1788,7 +1756,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="129" priority="1000000">
+ <synapse id="127" priority="1000000">
<action_set>
<pseudo_event id="55" operation="running" operation_key="rabbitmq-bundle_running_0">
<attributes CRM_meta_timeout="20000" />
@@ -1800,7 +1768,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="130">
+ <synapse id="128">
<action_set>
<pseudo_event id="54" operation="start" operation_key="rabbitmq-bundle_start_0">
<attributes CRM_meta_timeout="20000" />
@@ -1808,7 +1776,7 @@
</action_set>
<inputs/>
</synapse>
- <synapse id="131">
+ <synapse id="129">
<action_set>
<pseudo_event id="29" operation="all_stopped" operation_key="all_stopped">
<attributes />
@@ -1877,7 +1845,7 @@
</trigger>
</inputs>
</synapse>
- <synapse id="132">
+ <synapse id="130">
<action_set>
<crm_event id="1" operation="stonith" operation_key="stonith-controller-0-reboot" on_node="controller-0" on_node_uuid="1">
<attributes CRM_meta_cinder_volume_role="true" CRM_meta_galera_role="true" CRM_meta_haproxy_role="true" CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_rabbitmq_role="true" CRM_meta_redis_role="true" CRM_meta_rmq_node_attr_last_known_rabbitmq="rabbit@controller-0" CRM_meta_stonith_action="reboot" />
diff --git a/pengine/test10/bundle-order-fencing.summary b/pengine/test10/bundle-order-fencing.summary
index d398a12..a3dc3d4 100644
--- a/pengine/test10/bundle-order-fencing.summary
+++ b/pengine/test10/bundle-order-fencing.summary
@@ -174,7 +174,6 @@ Executing cluster transition:
* Pseudo action: redis-bundle_running_0
* Pseudo action: redis-bundle-master_pre_notify_promote_0
* Pseudo action: redis-bundle_promote_0
- * Resource action: redis notify on redis-bundle-0
* Resource action: redis notify on redis-bundle-1
* Resource action: redis notify on redis-bundle-2
* Pseudo action: redis-bundle-master_confirmed-pre_notify_promote_0
@@ -182,7 +181,6 @@ Executing cluster transition:
* Resource action: redis promote on redis-bundle-1
* Pseudo action: redis-bundle-master_promoted_0
* Pseudo action: redis-bundle-master_post_notify_promoted_0
- * Resource action: redis notify on redis-bundle-0
* Resource action: redis notify on redis-bundle-1
* Resource action: redis notify on redis-bundle-2
* Pseudo action: redis-bundle-master_confirmed-post_notify_promoted_0
--
1.8.3.1
From 48198ca839b62de1316d7ae6ab0994dedb37b523 Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 27 Nov 2018 17:00:25 -0600
Subject: [PATCH 4/5] Refactor: controller: make process_lrm_event() void
All callers ignored the return value
---
crmd/crmd_lrm.h | 3 ++-
crmd/lrm.c | 8 +++-----
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h
index d115056..3e1596d 100644
--- a/crmd/crmd_lrm.h
+++ b/crmd/crmd_lrm.h
@@ -170,4 +170,5 @@ gboolean remote_ra_is_in_maintenance(lrm_state_t * lrm_state);
void remote_ra_process_maintenance_nodes(xmlNode *xml);
gboolean remote_ra_controlling_guest(lrm_state_t * lrm_state);
-gboolean process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurring_op_s *pending);
+void process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
+ struct recurring_op_s *pending);
diff --git a/crmd/lrm.c b/crmd/lrm.c
index d18665c..5e5af9f 100644
--- a/crmd/lrm.c
+++ b/crmd/lrm.c
@@ -2515,7 +2515,7 @@ unescape_newlines(const char *string)
return ret;
}
-gboolean
+void
process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurring_op_s *pending)
{
char *op_id = NULL;
@@ -2526,8 +2526,8 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
gboolean removed = FALSE;
lrmd_rsc_info_t *rsc = NULL;
- CRM_CHECK(op != NULL, return FALSE);
- CRM_CHECK(op->rsc_id != NULL, return FALSE);
+ CRM_CHECK(op != NULL, return);
+ CRM_CHECK(op->rsc_id != NULL, return);
op_id = make_stop_id(op->rsc_id, op->call_id);
op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
@@ -2679,6 +2679,4 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
lrmd_free_rsc_info(rsc);
free(op_key);
free(op_id);
-
- return TRUE;
}
--
1.8.3.1
From 77dd44e214401d4dd953a8bafa2469b36d70948e Mon Sep 17 00:00:00 2001
From: Ken Gaillot <kgaillot@redhat.com>
Date: Tue, 27 Nov 2018 17:02:36 -0600
Subject: [PATCH 5/5] Low: controller: be more tolerant of malformed executor
events
b3f9a5bb was overzealous in discarding faked executor results without any
resource information. Since that commit, synthesize_lrmd_failure() would check
for resource information, and send a CIB update if the synthesized operation
were recordable, but would otherwise (such as for notifications) discard the
result.
This means the fix was complete, because non-recordable actions for a
resource behind a just-died remote connection would get lost. It also
exposed two pre-existing bugs regarding notifications mis-scheduled on
the wrong node. Any of these would block the transition from completing.
Now, process_lrm_event() can handle missing lrm_state or resource information,
so it can be called by synthesize_lrmd_failure() without any checking. This
leads to all the normal handling for non-recordable operations, which doesn't
require resource information. We log an assertion if the resource information
is not found, so that we can still get some visibility into bugs. This won't
be of use in the case of mis-scheduled notifications, but it could help in
other situations.
---
crmd/crmd_lrm.h | 2 +-
crmd/lrm.c | 148 ++++++++++++++++++++++++++++++--------------------
crmd/lrm_state.c | 2 +-
crmd/remote_lrmd_ra.c | 2 +-
4 files changed, 93 insertions(+), 61 deletions(-)
diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h
index 3e1596d..0870817 100644
--- a/crmd/crmd_lrm.h
+++ b/crmd/crmd_lrm.h
@@ -171,4 +171,4 @@ void remote_ra_process_maintenance_nodes(xmlNode *xml);
gboolean remote_ra_controlling_guest(lrm_state_t * lrm_state);
void process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
- struct recurring_op_s *pending);
+ struct recurring_op_s *pending, xmlNode *action_xml);
diff --git a/crmd/lrm.c b/crmd/lrm.c
index 5e5af9f..0d64f59 100644
--- a/crmd/lrm.c
+++ b/crmd/lrm.c
@@ -314,7 +314,7 @@ lrm_op_callback(lrmd_event_data_t * op)
lrm_state = lrm_state_find(nodename);
CRM_ASSERT(lrm_state != NULL);
- process_lrm_event(lrm_state, op, NULL);
+ process_lrm_event(lrm_state, op, NULL, NULL);
}
/* A_LRM_CONNECT */
@@ -1434,7 +1434,6 @@ static void
synthesize_lrmd_failure(lrm_state_t *lrm_state, xmlNode *action, int rc)
{
lrmd_event_data_t *op = NULL;
- lrmd_rsc_info_t *rsc_info = NULL;
const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK);
const char *target_node = crm_element_value(action, XML_LRM_ATTR_TARGET);
xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE);
@@ -1464,35 +1463,8 @@ synthesize_lrmd_failure(lrm_state_t *lrm_state, xmlNode *action, int rc)
crm_info("Faking %s_%s_%d result (%d) on %s",
op->rsc_id, op->op_type, op->interval, op->rc, target_node);
- /* Process the result as if it came from the LRM, if possible
- * (i.e. resource info can be obtained from the lrm_state).
- */
- if (lrm_state) {
- rsc_info = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
- }
- if (rsc_info) {
- lrmd_free_rsc_info(rsc_info);
- process_lrm_event(lrm_state, op, NULL);
-
- } else if (controld_action_is_recordable(op->op_type)) {
- /* If we can't process the result normally, at least write it to the CIB
- * if possible, so the PE can act on it.
- */
- const char *standard = crm_element_value(xml_rsc, XML_AGENT_ATTR_CLASS);
- const char *provider = crm_element_value(xml_rsc, XML_AGENT_ATTR_PROVIDER);
- const char *type = crm_element_value(xml_rsc, XML_ATTR_TYPE);
-
- if (standard && type) {
- rsc_info = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
- do_update_resource(target_node, rsc_info, op);
- lrmd_free_rsc_info(rsc_info);
- } else {
- // @TODO Should we direct ack?
- crm_info("Can't fake %s failure (%d) on %s without resource standard and type",
- crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc,
- target_node);
- }
- }
+ // Process the result as if it came from the LRM
+ process_lrm_event(lrm_state, op, NULL, action);
lrmd_free_event(op);
}
@@ -1555,7 +1527,7 @@ fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
crm_info("Failing resource %s...", rsc->id);
- process_lrm_event(lrm_state, op, NULL);
+ process_lrm_event(lrm_state, op, NULL, xml);
op->op_status = PCMK_LRM_OP_DONE;
op->rc = PCMK_OCF_OK;
lrmd_free_rsc_info(rsc);
@@ -2315,7 +2287,7 @@ do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *operat
crm_err("Operation %s on resource %s failed to execute on remote node %s: %d",
operation, rsc->id, lrm_state->node_name, call_id);
fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_UNKNOWN_ERROR);
- process_lrm_event(lrm_state, op, NULL);
+ process_lrm_event(lrm_state, op, NULL, NULL);
} else {
/* record all operations so we can wait
@@ -2516,7 +2488,8 @@ unescape_newlines(const char *string)
}
void
-process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurring_op_s *pending)
+process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
+ struct recurring_op_s *pending, xmlNode *action_xml)
{
char *op_id = NULL;
char *op_key = NULL;
@@ -2525,16 +2498,49 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
gboolean remove = FALSE;
gboolean removed = FALSE;
lrmd_rsc_info_t *rsc = NULL;
+ const char *node_name = NULL;
CRM_CHECK(op != NULL, return);
CRM_CHECK(op->rsc_id != NULL, return);
op_id = make_stop_id(op->rsc_id, op->call_id);
op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
- rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
+
+ // Get resource info if available (from executor state or action XML)
+ if (lrm_state) {
+ rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
+ }
+ if ((rsc == NULL) && action_xml) {
+ xmlNode *xml = find_xml_node(action_xml, XML_CIB_TAG_RESOURCE, TRUE);
+
+ const char *standard = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
+ const char *provider = crm_element_value(xml, XML_AGENT_ATTR_PROVIDER);
+ const char *type = crm_element_value(xml, XML_ATTR_TYPE);
+
+ if (standard && type) {
+ crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
+ op->rsc_id, standard,
+ (provider? ":" : ""), (provider? provider : ""), type);
+ rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
+ } else {
+ crm_err("Can't process %s result because %s agent information not cached or in XML",
+ op_key, op->rsc_id);
+ }
+ }
+ CRM_LOG_ASSERT(rsc != NULL); // If it's still NULL, there's a bug somewhere
+
+ // Get node name if available (from executor state or action XML)
+ if (lrm_state) {
+ node_name = lrm_state->node_name;
+ } else if (action_xml) {
+ node_name = crm_element_value(action_xml, XML_LRM_ATTR_TARGET);
+ }
+
if(pending == NULL) {
remove = TRUE;
- pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
+ if (lrm_state) {
+ pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
+ }
}
if (op->op_status == PCMK_LRM_OP_ERROR) {
@@ -2554,7 +2560,14 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
if (op->op_status != PCMK_LRM_OP_CANCELLED) {
if (controld_action_is_recordable(op->op_type)) {
- update_id = do_update_resource(lrm_state->node_name, rsc, op);
+ if (node_name && rsc) {
+ update_id = do_update_resource(node_name, rsc, op);
+ } else {
+ // @TODO Should we direct ack?
+ crm_err("Unable to record %s result in CIB: %s",
+ op_key,
+ (node_name? "No resource information" : "No node name"));
+ }
} else {
send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
}
@@ -2575,7 +2588,9 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
} else if (pending->remove) {
/* The tengine canceled this op, we have been waiting for the cancel to finish. */
- erase_lrm_history_by_op(lrm_state, op);
+ if (lrm_state) {
+ erase_lrm_history_by_op(lrm_state, op);
+ }
} else if (op->rsc_deleted) {
/* The tengine initiated this op, but it was cancelled outside of the
@@ -2595,14 +2610,23 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
/* The caller will do this afterwards, but keep the logging consistent */
removed = TRUE;
- } else if ((op->interval == 0) && g_hash_table_remove(lrm_state->pending_ops, op_id)) {
- removed = TRUE;
- crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
- op_key, op->call_id, op_id, g_hash_table_size(lrm_state->pending_ops));
+ } else if (lrm_state && ((op->interval == 0)
+ || (op->op_status == PCMK_LRM_OP_CANCELLED))) {
- } else if(op->interval != 0 && op->op_status == PCMK_LRM_OP_CANCELLED) {
- removed = TRUE;
- g_hash_table_remove(lrm_state->pending_ops, op_id);
+ gboolean found = g_hash_table_remove(lrm_state->pending_ops, op_id);
+
+ if (op->interval != 0) {
+ removed = TRUE;
+ } else if (found) {
+ removed = TRUE;
+ crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
+ op_key, op->call_id, op_id,
+ g_hash_table_size(lrm_state->pending_ops));
+ }
+ }
+
+ if (node_name == NULL) {
+ node_name = "unknown node"; // for logging
}
switch (op->op_status) {
@@ -2610,7 +2634,7 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
crm_info("Result of %s operation for %s on %s: %s "
CRM_XS " call=%d key=%s confirmed=%s",
crm_action_str(op->op_type, op->interval),
- op->rsc_id, lrm_state->node_name,
+ op->rsc_id, node_name,
services_lrm_status_str(op->op_status),
op->call_id, op_key, (removed? "true" : "false"));
break;
@@ -2620,7 +2644,7 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
"Result of %s operation for %s on %s: %d (%s) "
CRM_XS " call=%d key=%s confirmed=%s cib-update=%d",
crm_action_str(op->op_type, op->interval),
- op->rsc_id, lrm_state->node_name,
+ op->rsc_id, node_name,
op->rc, services_ocf_exitcode_str(op->rc),
op->call_id, op_key, (removed? "true" : "false"),
update_id);
@@ -2630,7 +2654,7 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
crm_err("Result of %s operation for %s on %s: %s "
CRM_XS " call=%d key=%s timeout=%dms",
crm_action_str(op->op_type, op->interval),
- op->rsc_id, lrm_state->node_name,
+ op->rsc_id, node_name,
services_lrm_status_str(op->op_status),
op->call_id, op_key, op->timeout);
break;
@@ -2639,14 +2663,16 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
crm_err("Result of %s operation for %s on %s: %s "
CRM_XS " call=%d key=%s confirmed=%s status=%d cib-update=%d",
crm_action_str(op->op_type, op->interval),
- op->rsc_id, lrm_state->node_name,
+ op->rsc_id, node_name,
services_lrm_status_str(op->op_status), op->call_id, op_key,
(removed? "true" : "false"), op->op_status, update_id);
}
if (op->output) {
char *prefix =
- crm_strdup_printf("%s-%s_%s_%d:%d", lrm_state->node_name, op->rsc_id, op->op_type, op->interval, op->call_id);
+ crm_strdup_printf("%s-%s_%s_%d:%d", node_name,
+ op->rsc_id, op->op_type, op->interval,
+ op->call_id);
if (op->rc) {
crm_log_output(LOG_NOTICE, prefix, op->output);
@@ -2656,25 +2682,31 @@ process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurr
free(prefix);
}
- if (safe_str_neq(op->op_type, RSC_METADATA)) {
- crmd_alert_resource_op(lrm_state->node_name, op);
- } else if (op->rc == PCMK_OCF_OK) {
- char *metadata = unescape_newlines(op->output);
+ if (lrm_state) {
+ if (safe_str_neq(op->op_type, RSC_METADATA)) {
+ crmd_alert_resource_op(lrm_state->node_name, op);
+ } else if (rsc && (op->rc == PCMK_OCF_OK)) {
+ char *metadata = unescape_newlines(op->output);
- metadata_cache_update(lrm_state->metadata_cache, rsc, metadata);
- free(metadata);
+ metadata_cache_update(lrm_state->metadata_cache, rsc, metadata);
+ free(metadata);
+ }
}
if (op->rsc_deleted) {
crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
- delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
+ if (lrm_state) {
+ delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
+ }
}
/* If a shutdown was escalated while operations were pending,
* then the FSA will be stalled right now... allow it to continue
*/
mainloop_set_trigger(fsa_source);
- update_history_cache(lrm_state, rsc, op);
+ if (lrm_state && rsc) {
+ update_history_cache(lrm_state, rsc, op);
+ }
lrmd_free_rsc_info(rsc);
free(op_key);
diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c
index 40da762..d8a0039 100644
--- a/crmd/lrm_state.c
+++ b/crmd/lrm_state.c
@@ -96,7 +96,7 @@ fail_pending_op(gpointer key, gpointer value, gpointer user_data)
event.remote_nodename = lrm_state->node_name;
event.params = op->params;
- process_lrm_event(lrm_state, &event, op);
+ process_lrm_event(lrm_state, &event, op, NULL);
return TRUE;
}
diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c
index 6fa05f6..2d04588 100644
--- a/crmd/remote_lrmd_ra.c
+++ b/crmd/remote_lrmd_ra.c
@@ -519,7 +519,7 @@ synthesize_lrmd_success(lrm_state_t *lrm_state, const char *rsc_id, const char *
op.t_run = time(NULL);
op.t_rcchange = op.t_run;
op.call_id = generate_callid();
- process_lrm_event(lrm_state, &op, NULL);
+ process_lrm_event(lrm_state, &op, NULL, NULL);
}
void
--
1.8.3.1