Blame SOURCES/060-bundle-remote-fixes.patch

60de42
From 7a9d55107fd3cd7c7f11854e8bf06f35900ded98 Mon Sep 17 00:00:00 2001
60de42
From: Andrew Beekhof <andrew@beekhof.net>
60de42
Date: Fri, 19 May 2017 13:55:51 +1000
60de42
Subject: [PATCH 01/13] Fix: PE: Partially restore 62ed004 to ensure remote
60de42
 connections are available before attempting resource recovery
60de42
60de42
However only do so for non-container connections
60de42
---
60de42
 pengine/allocate.c | 9 +++++++++
60de42
 1 file changed, 9 insertions(+)
60de42
60de42
diff --git a/pengine/allocate.c b/pengine/allocate.c
60de42
index 3314d77..3819c24 100644
60de42
--- a/pengine/allocate.c
60de42
+++ b/pengine/allocate.c
60de42
@@ -1836,6 +1836,15 @@ apply_remote_node_ordering(pe_working_set_t *data_set)
60de42
                     NULL,
60de42
                     pe_order_preserve | pe_order_implies_first,
60de42
                     data_set);
60de42
+            } else if(container == NULL) {
60de42
+                custom_action_order(remote_rsc,
60de42
+                    generate_op_key(remote_rsc->id, RSC_START, 0),
60de42
+                    NULL,
60de42
+                    action->rsc,
60de42
+                    NULL,
60de42
+                    action,
60de42
+                    pe_order_preserve | pe_order_implies_then | pe_order_runnable_left,
60de42
+                    data_set);
60de42
             }
60de42
 
60de42
             if(container && is_set(container->flags, pe_rsc_failed)) {
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From 4d52523d6b101226a6bf71fe009c98df15c7ad3e Mon Sep 17 00:00:00 2001
60de42
From: Andrew Beekhof <andrew@beekhof.net>
60de42
Date: Fri, 19 May 2017 13:59:50 +1000
60de42
Subject: [PATCH 02/13] Test: PE: Ensure remote connections are available
60de42
 before attempting resource recovery
60de42
60de42
---
60de42
 pengine/regression.sh                  |    1 +
60de42
 pengine/test10/remote-recovery.dot     |  221 +++++++
60de42
 pengine/test10/remote-recovery.exp     | 1118 ++++++++++++++++++++++++++++++++
60de42
 pengine/test10/remote-recovery.scores  |  848 ++++++++++++++++++++++++
60de42
 pengine/test10/remote-recovery.summary |  164 +++++
60de42
 pengine/test10/remote-recovery.xml     |  739 +++++++++++++++++++++
60de42
 6 files changed, 3091 insertions(+)
60de42
 create mode 100644 pengine/test10/remote-recovery.dot
60de42
 create mode 100644 pengine/test10/remote-recovery.exp
60de42
 create mode 100644 pengine/test10/remote-recovery.scores
60de42
 create mode 100644 pengine/test10/remote-recovery.summary
60de42
 create mode 100644 pengine/test10/remote-recovery.xml
60de42
60de42
diff --git a/pengine/regression.sh b/pengine/regression.sh
60de42
index e2e7bf6..62fc066 100755
60de42
--- a/pengine/regression.sh
60de42
+++ b/pengine/regression.sh
60de42
@@ -837,6 +837,7 @@ do_test remote-recover-fail     "Make sure start failure causes fencing if rsc a
60de42
 do_test remote-start-fail       "Make sure a start failure does not result in fencing if no active resources are on remote."
60de42
 do_test remote-unclean2         "Make monitor failure always results in fencing, even if no rsc are active on remote."
60de42
 do_test remote-fence-before-reconnect "Fence before clearing recurring monitor failure"
60de42
+do_test remote-recovery		"Recover remote connections before attempting demotion"
60de42
 
60de42
 echo ""
60de42
 do_test resource-discovery      "Exercises resource-discovery location constraint option."
60de42
diff --git a/pengine/test10/remote-recovery.dot b/pengine/test10/remote-recovery.dot
60de42
new file mode 100644
60de42
index 0000000..31a82af
60de42
--- /dev/null
60de42
+++ b/pengine/test10/remote-recovery.dot
60de42
@@ -0,0 +1,221 @@
60de42
+digraph "g" {
60de42
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
60de42
+"galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold]
60de42
+"galera-0_start_0 controller-2" -> "galera_demote_0 galera-0" [ style = bold]
60de42
+"galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold]
60de42
+"galera-0_start_0 controller-2" -> "galera_promote_0 galera-0" [ style = bold]
60de42
+"galera-0_start_0 controller-2" -> "galera_start_0 galera-0" [ style = bold]
60de42
+"galera-0_start_0 controller-2" -> "galera_stop_0 galera-0" [ style = bold]
60de42
+"galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"galera-0_stop_0 controller-1" -> "all_stopped" [ style = bold]
60de42
+"galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold]
60de42
+"galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
60de42
+"galera-2_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"galera-2_start_0 controller-2" -> "galera-2_monitor_20000 controller-2" [ style = bold]
60de42
+"galera-2_start_0 controller-2" -> "galera_demote_0 galera-2" [ style = bold]
60de42
+"galera-2_start_0 controller-2" -> "galera_monitor_10000 galera-2" [ style = bold]
60de42
+"galera-2_start_0 controller-2" -> "galera_promote_0 galera-2" [ style = bold]
60de42
+"galera-2_start_0 controller-2" -> "galera_start_0 galera-2" [ style = bold]
60de42
+"galera-2_start_0 controller-2" -> "galera_stop_0 galera-2" [ style = bold]
60de42
+"galera-2_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"galera-2_stop_0 controller-1" -> "all_stopped" [ style = bold]
60de42
+"galera-2_stop_0 controller-1" -> "galera-2_start_0 controller-2" [ style = bold]
60de42
+"galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
60de42
+"galera-master_demote_0" -> "galera-master_demoted_0" [ style = bold]
60de42
+"galera-master_demote_0" -> "galera_demote_0 galera-0" [ style = bold]
60de42
+"galera-master_demote_0" -> "galera_demote_0 galera-2" [ style = bold]
60de42
+"galera-master_demote_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"galera-master_demoted_0" -> "galera-master_promote_0" [ style = bold]
60de42
+"galera-master_demoted_0" -> "galera-master_start_0" [ style = bold]
60de42
+"galera-master_demoted_0" -> "galera-master_stop_0" [ style = bold]
60de42
+"galera-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"galera-master_promote_0" -> "galera_promote_0 galera-0" [ style = bold]
60de42
+"galera-master_promote_0" -> "galera_promote_0 galera-2" [ style = bold]
60de42
+"galera-master_promote_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"galera-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"galera-master_running_0" -> "galera-master_promote_0" [ style = bold]
60de42
+"galera-master_running_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"galera-master_start_0" -> "galera-master_running_0" [ style = bold]
60de42
+"galera-master_start_0" -> "galera_start_0 galera-0" [ style = bold]
60de42
+"galera-master_start_0" -> "galera_start_0 galera-2" [ style = bold]
60de42
+"galera-master_start_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"galera-master_stop_0" -> "galera-master_stopped_0" [ style = bold]
60de42
+"galera-master_stop_0" -> "galera_stop_0 galera-0" [ style = bold]
60de42
+"galera-master_stop_0" -> "galera_stop_0 galera-2" [ style = bold]
60de42
+"galera-master_stop_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"galera-master_stopped_0" -> "galera-master_promote_0" [ style = bold]
60de42
+"galera-master_stopped_0" -> "galera-master_start_0" [ style = bold]
60de42
+"galera-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"galera_demote_0 galera-0" -> "galera-master_demoted_0" [ style = bold]
60de42
+"galera_demote_0 galera-0" -> "galera_demote_0 galera-2" [ style = bold]
60de42
+"galera_demote_0 galera-0" -> "galera_promote_0 galera-0" [ style = bold]
60de42
+"galera_demote_0 galera-0" -> "galera_stop_0 galera-0" [ style = bold]
60de42
+"galera_demote_0 galera-0" [ style=bold color="green" fontcolor="black"]
60de42
+"galera_demote_0 galera-2" -> "galera-master_demoted_0" [ style = bold]
60de42
+"galera_demote_0 galera-2" -> "galera_promote_0 galera-2" [ style = bold]
60de42
+"galera_demote_0 galera-2" -> "galera_stop_0 galera-2" [ style = bold]
60de42
+"galera_demote_0 galera-2" [ style=bold color="green" fontcolor="black"]
60de42
+"galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"]
60de42
+"galera_monitor_10000 galera-2" [ style=bold color="green" fontcolor="black"]
60de42
+"galera_promote_0 galera-0" -> "galera-master_promoted_0" [ style = bold]
60de42
+"galera_promote_0 galera-0" -> "galera_monitor_10000 galera-0" [ style = bold]
60de42
+"galera_promote_0 galera-0" [ style=bold color="green" fontcolor="black"]
60de42
+"galera_promote_0 galera-2" -> "galera-master_promoted_0" [ style = bold]
60de42
+"galera_promote_0 galera-2" -> "galera_monitor_10000 galera-2" [ style = bold]
60de42
+"galera_promote_0 galera-2" -> "galera_promote_0 galera-0" [ style = bold]
60de42
+"galera_promote_0 galera-2" [ style=bold color="green" fontcolor="black"]
60de42
+"galera_start_0 galera-0" -> "galera-master_running_0" [ style = bold]
60de42
+"galera_start_0 galera-0" -> "galera_monitor_10000 galera-0" [ style = bold]
60de42
+"galera_start_0 galera-0" -> "galera_promote_0 galera-0" [ style = bold]
60de42
+"galera_start_0 galera-0" [ style=bold color="green" fontcolor="black"]
60de42
+"galera_start_0 galera-2" -> "galera-master_running_0" [ style = bold]
60de42
+"galera_start_0 galera-2" -> "galera_monitor_10000 galera-2" [ style = bold]
60de42
+"galera_start_0 galera-2" -> "galera_promote_0 galera-2" [ style = bold]
60de42
+"galera_start_0 galera-2" -> "galera_start_0 galera-0" [ style = bold]
60de42
+"galera_start_0 galera-2" [ style=bold color="green" fontcolor="black"]
60de42
+"galera_stop_0 galera-0" -> "all_stopped" [ style = bold]
60de42
+"galera_stop_0 galera-0" -> "galera-master_stopped_0" [ style = bold]
60de42
+"galera_stop_0 galera-0" -> "galera_start_0 galera-0" [ style = bold]
60de42
+"galera_stop_0 galera-0" -> "galera_stop_0 galera-2" [ style = bold]
60de42
+"galera_stop_0 galera-0" [ style=bold color="green" fontcolor="black"]
60de42
+"galera_stop_0 galera-2" -> "all_stopped" [ style = bold]
60de42
+"galera_stop_0 galera-2" -> "galera-master_stopped_0" [ style = bold]
60de42
+"galera_stop_0 galera-2" -> "galera_start_0 galera-2" [ style = bold]
60de42
+"galera_stop_0 galera-2" [ style=bold color="green" fontcolor="black"]
60de42
+"haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold]
60de42
+"haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold]
60de42
+"haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
60de42
+"haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
60de42
+"haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
60de42
+"haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"haproxy_stop_0 controller-1" -> "all_stopped" [ style = bold]
60de42
+"haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold]
60de42
+"haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
60de42
+"ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold]
60de42
+"ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"ip-172.17.1.14_stop_0 controller-1" -> "all_stopped" [ style = bold]
60de42
+"ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
60de42
+"ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
60de42
+"ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold]
60de42
+"ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"ip-172.17.1.17_stop_0 controller-1" -> "all_stopped" [ style = bold]
60de42
+"ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
60de42
+"ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
60de42
+"ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold]
60de42
+"ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"ip-172.17.4.11_stop_0 controller-1" -> "all_stopped" [ style = bold]
60de42
+"ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
60de42
+"ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
60de42
+"messaging-1_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"messaging-1_start_0 controller-2" -> "messaging-1_monitor_20000 controller-2" [ style = bold]
60de42
+"messaging-1_start_0 controller-2" -> "rabbitmq_monitor_10000 messaging-1" [ style = bold]
60de42
+"messaging-1_start_0 controller-2" -> "rabbitmq_start_0 messaging-1" [ style = bold]
60de42
+"messaging-1_start_0 controller-2" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
60de42
+"messaging-1_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"messaging-1_stop_0 controller-1" -> "all_stopped" [ style = bold]
60de42
+"messaging-1_stop_0 controller-1" -> "messaging-1_start_0 controller-2" [ style = bold]
60de42
+"messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
60de42
+"rabbitmq-clone_running_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"rabbitmq-clone_start_0" -> "rabbitmq-clone_running_0" [ style = bold]
60de42
+"rabbitmq-clone_start_0" -> "rabbitmq_start_0 messaging-1" [ style = bold]
60de42
+"rabbitmq-clone_start_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold]
60de42
+"rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
60de42
+"rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"rabbitmq-clone_stopped_0" -> "rabbitmq-clone_start_0" [ style = bold]
60de42
+"rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"rabbitmq_monitor_10000 messaging-1" [ style=bold color="green" fontcolor="black"]
60de42
+"rabbitmq_start_0 messaging-1" -> "rabbitmq-clone_running_0" [ style = bold]
60de42
+"rabbitmq_start_0 messaging-1" -> "rabbitmq_monitor_10000 messaging-1" [ style = bold]
60de42
+"rabbitmq_start_0 messaging-1" [ style=bold color="green" fontcolor="black"]
60de42
+"rabbitmq_stop_0 messaging-1" -> "all_stopped" [ style = bold]
60de42
+"rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold]
60de42
+"rabbitmq_stop_0 messaging-1" -> "rabbitmq_start_0 messaging-1" [ style = bold]
60de42
+"rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="black"]
60de42
+"redis-master_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold]
60de42
+"redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
60de42
+"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold]
60de42
+"redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
60de42
+"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
60de42
+"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
60de42
+"redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
60de42
+"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold]
60de42
+"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold]
60de42
+"redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold]
60de42
+"redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold]
60de42
+"redis-master_stop_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
60de42
+"redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"redis_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold]
60de42
+"redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
60de42
+"redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
60de42
+"redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"]
60de42
+"redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
60de42
+"redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
60de42
+"redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
60de42
+"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
60de42
+"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
60de42
+"redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
60de42
+"redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
60de42
+"redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
60de42
+"redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
60de42
+"redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"redis_stop_0 controller-1" -> "all_stopped" [ style = bold]
60de42
+"redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold]
60de42
+"redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
60de42
+"stonith 'reboot' controller-1" -> "galera-0_stop_0 controller-1" [ style = bold]
60de42
+"stonith 'reboot' controller-1" -> "galera-2_stop_0 controller-1" [ style = bold]
60de42
+"stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold]
60de42
+"stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold]
60de42
+"stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
60de42
+"stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
60de42
+"stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
60de42
+"stonith 'reboot' controller-1" -> "messaging-1_stop_0 controller-1" [ style = bold]
60de42
+"stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold]
60de42
+"stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold]
60de42
+"stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold]
60de42
+"stonith 'reboot' controller-1" -> "stonith_complete" [ style = bold]
60de42
+"stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"]
60de42
+"stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold]
60de42
+"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
60de42
+"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "all_stopped" [ style = bold]
60de42
+"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
60de42
+"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
60de42
+"stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"]
60de42
+"stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold]
60de42
+"stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"]
60de42
+"stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "all_stopped" [ style = bold]
60de42
+"stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold]
60de42
+"stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
60de42
+"stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"]
60de42
+"stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold]
60de42
+"stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"]
60de42
+"stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "all_stopped" [ style = bold]
60de42
+"stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold]
60de42
+"stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
60de42
+"stonith_complete" -> "all_stopped" [ style = bold]
60de42
+"stonith_complete" -> "galera-0_start_0 controller-2" [ style = bold]
60de42
+"stonith_complete" -> "galera-2_start_0 controller-2" [ style = bold]
60de42
+"stonith_complete" -> "galera_promote_0 galera-0" [ style = bold]
60de42
+"stonith_complete" -> "galera_promote_0 galera-2" [ style = bold]
60de42
+"stonith_complete" -> "galera_start_0 galera-0" [ style = bold]
60de42
+"stonith_complete" -> "galera_start_0 galera-2" [ style = bold]
60de42
+"stonith_complete" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
60de42
+"stonith_complete" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
60de42
+"stonith_complete" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
60de42
+"stonith_complete" -> "messaging-1_start_0 controller-2" [ style = bold]
60de42
+"stonith_complete" -> "rabbitmq_start_0 messaging-1" [ style = bold]
60de42
+"stonith_complete" [ style=bold color="green" fontcolor="orange"]
60de42
+}
60de42
diff --git a/pengine/test10/remote-recovery.exp b/pengine/test10/remote-recovery.exp
60de42
new file mode 100644
60de42
index 0000000..70492f4
60de42
--- /dev/null
60de42
+++ b/pengine/test10/remote-recovery.exp
60de42
@@ -0,0 +1,1118 @@
60de42
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
60de42
+  <synapse id="0">
60de42
+    <action_set>
60de42
+      <rsc_op id="28" operation="monitor" operation_key="messaging-1_monitor_20000" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="messaging-1" class="ocf" provider="pacemaker" type="remote"/>
60de42
+        <attributes CRM_meta_interval="20000" CRM_meta_name="monitor" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000"  reconnect_interval="60"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="27" operation="start" operation_key="messaging-1_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="1">
60de42
+    <action_set>
60de42
+      <rsc_op id="27" operation="start" operation_key="messaging-1_start_0" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="messaging-1" class="ocf" provider="pacemaker" type="remote"/>
60de42
+        <attributes CRM_meta_name="start" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="60000"  reconnect_interval="60"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="26" operation="stop" operation_key="messaging-1_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="2">
60de42
+    <action_set>
60de42
+      <pseudo_event id="26" operation="stop" operation_key="messaging-1_stop_0">
60de42
+        <attributes CRM_meta_name="stop" CRM_meta_timeout="60000"  reconnect_interval="60"/>
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="3">
60de42
+    <action_set>
60de42
+      <rsc_op id="33" operation="monitor" operation_key="galera-0_monitor_20000" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="galera-0" class="ocf" provider="pacemaker" type="remote"/>
60de42
+        <attributes CRM_meta_interval="20000" CRM_meta_name="monitor" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000"  reconnect_interval="60"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="32" operation="start" operation_key="galera-0_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="4">
60de42
+    <action_set>
60de42
+      <rsc_op id="32" operation="start" operation_key="galera-0_start_0" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="galera-0" class="ocf" provider="pacemaker" type="remote"/>
60de42
+        <attributes CRM_meta_name="start" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="60000"  reconnect_interval="60"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="31" operation="stop" operation_key="galera-0_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="5">
60de42
+    <action_set>
60de42
+      <pseudo_event id="31" operation="stop" operation_key="galera-0_stop_0">
60de42
+        <attributes CRM_meta_name="stop" CRM_meta_timeout="60000"  reconnect_interval="60"/>
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="6">
60de42
+    <action_set>
60de42
+      <rsc_op id="38" operation="monitor" operation_key="galera-2_monitor_20000" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="galera-2" class="ocf" provider="pacemaker" type="remote"/>
60de42
+        <attributes CRM_meta_interval="20000" CRM_meta_name="monitor" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000"  reconnect_interval="60"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="37" operation="start" operation_key="galera-2_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="7">
60de42
+    <action_set>
60de42
+      <rsc_op id="37" operation="start" operation_key="galera-2_start_0" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="galera-2" class="ocf" provider="pacemaker" type="remote"/>
60de42
+        <attributes CRM_meta_name="start" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="60000"  reconnect_interval="60"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="36" operation="stop" operation_key="galera-2_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="8">
60de42
+    <action_set>
60de42
+      <pseudo_event id="36" operation="stop" operation_key="galera-2_stop_0">
60de42
+        <attributes CRM_meta_name="stop" CRM_meta_timeout="60000"  reconnect_interval="60"/>
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="9">
60de42
+    <action_set>
60de42
+      <rsc_op id="44" operation="start" operation_key="rabbitmq_start_0" internal_operation_key="rabbitmq:2_start_0" on_node="messaging-1" on_node_uuid="messaging-1" router_node="controller-2">
60de42
+        <primitive id="rabbitmq" long-id="rabbitmq:2" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
60de42
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="messaging-1" CRM_meta_on_node_uuid="messaging-1" CRM_meta_timeout="200000"  set_policy="ha-all ^(?!amq\.).* {"ha-mode":"exactly","ha-params":2}"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="27" operation="start" operation_key="messaging-1_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="43" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:2_stop_0" on_node="messaging-1" on_node_uuid="messaging-1" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="45" operation="start" operation_key="rabbitmq-clone_start_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="10">
60de42
+    <action_set>
60de42
+      <rsc_op id="43" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:2_stop_0" on_node="messaging-1" on_node_uuid="messaging-1" router_node="controller-1">
60de42
+        <primitive id="rabbitmq" long-id="rabbitmq:2" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
60de42
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="messaging-1" CRM_meta_on_node_uuid="messaging-1" CRM_meta_timeout="200000"  set_policy="ha-all ^(?!amq\.).* {"ha-mode":"exactly","ha-params":2}"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="27" operation="start" operation_key="messaging-1_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="47" operation="stop" operation_key="rabbitmq-clone_stop_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="11">
60de42
+    <action_set>
60de42
+      <rsc_op id="20" operation="monitor" operation_key="rabbitmq_monitor_10000" internal_operation_key="rabbitmq:2_monitor_10000" on_node="messaging-1" on_node_uuid="messaging-1" router_node="controller-2">
60de42
+        <primitive id="rabbitmq" long-id="rabbitmq:2" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
60de42
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="messaging-1" CRM_meta_on_node_uuid="messaging-1" CRM_meta_timeout="40000"  set_policy="ha-all ^(?!amq\.).* {"ha-mode":"exactly","ha-params":2}"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="27" operation="start" operation_key="messaging-1_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="44" operation="start" operation_key="rabbitmq_start_0" internal_operation_key="rabbitmq:2_start_0" on_node="messaging-1" on_node_uuid="messaging-1" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="12" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="48" operation="stopped" operation_key="rabbitmq-clone_stopped_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="43" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:2_stop_0" on_node="messaging-1" on_node_uuid="messaging-1" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="47" operation="stop" operation_key="rabbitmq-clone_stop_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="13">
60de42
+    <action_set>
60de42
+      <pseudo_event id="47" operation="stop" operation_key="rabbitmq-clone_stop_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs/>
60de42
+  </synapse>
60de42
+  <synapse id="14" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="46" operation="running" operation_key="rabbitmq-clone_running_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="44" operation="start" operation_key="rabbitmq_start_0" internal_operation_key="rabbitmq:2_start_0" on_node="messaging-1" on_node_uuid="messaging-1" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="45" operation="start" operation_key="rabbitmq-clone_start_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="15">
60de42
+    <action_set>
60de42
+      <pseudo_event id="45" operation="start" operation_key="rabbitmq-clone_start_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="48" operation="stopped" operation_key="rabbitmq-clone_stopped_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="16">
60de42
+    <action_set>
60de42
+      <rsc_op id="56" operation="promote" operation_key="galera_promote_0" internal_operation_key="galera:1_promote_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-2">
60de42
+        <primitive id="galera" long-id="galera:1" class="ocf" provider="heartbeat" type="galera"/>
60de42
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_fail="block" CRM_meta_on_node="galera-2" CRM_meta_on_node_uuid="galera-2" CRM_meta_timeout="300000" additional_parameters="--open-files-limit=16384"  enable_creation="true" wsrep_cluster_address="gcomm://galera-0,galera-1,galera-2"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="37" operation="start" operation_key="galera-2_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="53" operation="start" operation_key="galera_start_0" internal_operation_key="galera:1_start_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="54" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:1_demote_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="65" operation="promote" operation_key="galera-master_promote_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="17">
60de42
+    <action_set>
60de42
+      <rsc_op id="55" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:1_stop_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-1">
60de42
+        <primitive id="galera" long-id="galera:1" class="ocf" provider="heartbeat" type="galera"/>
60de42
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="galera-2" CRM_meta_on_node_uuid="galera-2" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384"  enable_creation="true" wsrep_cluster_address="gcomm://galera-0,galera-1,galera-2"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="37" operation="start" operation_key="galera-2_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="54" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:1_demote_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="59" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:2_stop_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="63" operation="stop" operation_key="galera-master_stop_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="18">
60de42
+    <action_set>
60de42
+      <rsc_op id="54" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:1_demote_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-1">
60de42
+        <primitive id="galera" long-id="galera:1" class="ocf" provider="heartbeat" type="galera"/>
60de42
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="galera-2" CRM_meta_on_node_uuid="galera-2" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384"  enable_creation="true" wsrep_cluster_address="gcomm://galera-0,galera-1,galera-2"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="37" operation="start" operation_key="galera-2_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="58" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:2_demote_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="67" operation="demote" operation_key="galera-master_demote_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="19">
60de42
+    <action_set>
60de42
+      <rsc_op id="53" operation="start" operation_key="galera_start_0" internal_operation_key="galera:1_start_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-2">
60de42
+        <primitive id="galera" long-id="galera:1" class="ocf" provider="heartbeat" type="galera"/>
60de42
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="galera-2" CRM_meta_on_node_uuid="galera-2" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384"  enable_creation="true" wsrep_cluster_address="gcomm://galera-0,galera-1,galera-2"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="37" operation="start" operation_key="galera-2_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="55" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:1_stop_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="61" operation="start" operation_key="galera-master_start_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="20">
60de42
+    <action_set>
60de42
+      <rsc_op id="13" operation="monitor" operation_key="galera_monitor_10000" internal_operation_key="galera:1_monitor_10000" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-2">
60de42
+        <primitive id="galera" long-id="galera:1" class="ocf" provider="heartbeat" type="galera"/>
60de42
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-2" CRM_meta_on_node_uuid="galera-2" CRM_meta_op_target_rc="8" CRM_meta_role="Master" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384"  enable_creation="true" wsrep_cluster_address="gcomm://galera-0,galera-1,galera-2"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="37" operation="start" operation_key="galera-2_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="53" operation="start" operation_key="galera_start_0" internal_operation_key="galera:1_start_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="56" operation="promote" operation_key="galera_promote_0" internal_operation_key="galera:1_promote_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="21">
60de42
+    <action_set>
60de42
+      <rsc_op id="60" operation="promote" operation_key="galera_promote_0" internal_operation_key="galera:2_promote_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-2">
60de42
+        <primitive id="galera" long-id="galera:2" class="ocf" provider="heartbeat" type="galera"/>
60de42
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_fail="block" CRM_meta_on_node="galera-0" CRM_meta_on_node_uuid="galera-0" CRM_meta_timeout="300000" additional_parameters="--open-files-limit=16384"  enable_creation="true" wsrep_cluster_address="gcomm://galera-0,galera-1,galera-2"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="32" operation="start" operation_key="galera-0_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="56" operation="promote" operation_key="galera_promote_0" internal_operation_key="galera:1_promote_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="57" operation="start" operation_key="galera_start_0" internal_operation_key="galera:2_start_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="58" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:2_demote_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="65" operation="promote" operation_key="galera-master_promote_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="22">
60de42
+    <action_set>
60de42
+      <rsc_op id="59" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:2_stop_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-1">
60de42
+        <primitive id="galera" long-id="galera:2" class="ocf" provider="heartbeat" type="galera"/>
60de42
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="galera-0" CRM_meta_on_node_uuid="galera-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384"  enable_creation="true" wsrep_cluster_address="gcomm://galera-0,galera-1,galera-2"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="32" operation="start" operation_key="galera-0_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="58" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:2_demote_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="63" operation="stop" operation_key="galera-master_stop_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="23">
60de42
+    <action_set>
60de42
+      <rsc_op id="58" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:2_demote_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-1">
60de42
+        <primitive id="galera" long-id="galera:2" class="ocf" provider="heartbeat" type="galera"/>
60de42
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="galera-0" CRM_meta_on_node_uuid="galera-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384"  enable_creation="true" wsrep_cluster_address="gcomm://galera-0,galera-1,galera-2"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="32" operation="start" operation_key="galera-0_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="67" operation="demote" operation_key="galera-master_demote_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="24">
60de42
+    <action_set>
60de42
+      <rsc_op id="57" operation="start" operation_key="galera_start_0" internal_operation_key="galera:2_start_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-2">
60de42
+        <primitive id="galera" long-id="galera:2" class="ocf" provider="heartbeat" type="galera"/>
60de42
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="galera-0" CRM_meta_on_node_uuid="galera-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384"  enable_creation="true" wsrep_cluster_address="gcomm://galera-0,galera-1,galera-2"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="32" operation="start" operation_key="galera-0_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="53" operation="start" operation_key="galera_start_0" internal_operation_key="galera:1_start_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="59" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:2_stop_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="61" operation="start" operation_key="galera-master_start_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="25">
60de42
+    <action_set>
60de42
+      <rsc_op id="14" operation="monitor" operation_key="galera_monitor_10000" internal_operation_key="galera:2_monitor_10000" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-2">
60de42
+        <primitive id="galera" long-id="galera:2" class="ocf" provider="heartbeat" type="galera"/>
60de42
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-0" CRM_meta_on_node_uuid="galera-0" CRM_meta_op_target_rc="8" CRM_meta_role="Master" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384"  enable_creation="true" wsrep_cluster_address="gcomm://galera-0,galera-1,galera-2"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="32" operation="start" operation_key="galera-0_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="57" operation="start" operation_key="galera_start_0" internal_operation_key="galera:2_start_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="60" operation="promote" operation_key="galera_promote_0" internal_operation_key="galera:2_promote_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="26" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="68" operation="demoted" operation_key="galera-master_demoted_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="54" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:1_demote_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="58" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:2_demote_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="67" operation="demote" operation_key="galera-master_demote_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="27">
60de42
+    <action_set>
60de42
+      <pseudo_event id="67" operation="demote" operation_key="galera-master_demote_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs/>
60de42
+  </synapse>
60de42
+  <synapse id="28" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="66" operation="promoted" operation_key="galera-master_promoted_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="56" operation="promote" operation_key="galera_promote_0" internal_operation_key="galera:1_promote_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="60" operation="promote" operation_key="galera_promote_0" internal_operation_key="galera:2_promote_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="29">
60de42
+    <action_set>
60de42
+      <pseudo_event id="65" operation="promote" operation_key="galera-master_promote_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="62" operation="running" operation_key="galera-master_running_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="64" operation="stopped" operation_key="galera-master_stopped_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="68" operation="demoted" operation_key="galera-master_demoted_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="30" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="64" operation="stopped" operation_key="galera-master_stopped_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="55" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:1_stop_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="59" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:2_stop_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="63" operation="stop" operation_key="galera-master_stop_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="31">
60de42
+    <action_set>
60de42
+      <pseudo_event id="63" operation="stop" operation_key="galera-master_stop_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="68" operation="demoted" operation_key="galera-master_demoted_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="32" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="62" operation="running" operation_key="galera-master_running_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="53" operation="start" operation_key="galera_start_0" internal_operation_key="galera:1_start_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="57" operation="start" operation_key="galera_start_0" internal_operation_key="galera:2_start_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-2"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="61" operation="start" operation_key="galera-master_start_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="33">
60de42
+    <action_set>
60de42
+      <pseudo_event id="61" operation="start" operation_key="galera-master_start_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="64" operation="stopped" operation_key="galera-master_stopped_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="68" operation="demoted" operation_key="galera-master_demoted_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="34" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="134" operation="notified" operation_key="redis_notified_0" internal_operation_key="redis:0_confirmed-post_notify_stonith_0">
60de42
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stonith" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000"  wait_last_known_master="true"/>
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="133" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:0_post_notify_stonith_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="135" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:1_post_notify_stop_0" on_node="controller-0" on_node_uuid="1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="136" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:2_post_notify_stop_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="35" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="133" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:0_post_notify_stonith_0">
60de42
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stonith" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000"  wait_last_known_master="true"/>
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="36">
60de42
+    <action_set>
60de42
+      <pseudo_event id="69" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0">
60de42
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="controller-1 controller-0 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2" CRM_meta_notify_available_uname="controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:3 redis:4 redis:5 redis:6 redis:7 redis:8" CRM_meta_notify_master_resource="redis:1" CRM_meta_notify_master_uname="controller-0" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="controller-1 controller-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="controller-1" CRM_meta_timeout="200000"  wait_last_known_master="true"/>
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="82" operation="stop" operation_key="redis-master_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="37">
60de42
+    <action_set>
60de42
+      <rsc_op id="193" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:1_pre_notify_stop_0" on_node="controller-0" on_node_uuid="1">
60de42
+        <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
60de42
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="controller-1 controller-0 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2" CRM_meta_notify_available_uname="controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:3 redis:4 redis:5 redis:6 redis:7 redis:8" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:1" CRM_meta_notify_master_uname="controller-0" CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="controller-1 controller-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="controller-1" CRM_meta_notify_type="pre" CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000"  wait_last_known_master="true"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="84" operation="notify" operation_key="redis-master_pre_notify_stop_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="38" priority="1000000">
60de42
+    <action_set>
60de42
+      <rsc_op id="135" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:1_post_notify_stop_0" on_node="controller-0" on_node_uuid="1">
60de42
+        <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
60de42
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="controller-1 controller-0 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2" CRM_meta_notify_available_uname="controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:3 redis:4 redis:5 redis:6 redis:7 redis:8" CRM_meta_notify_key_operation="stonith" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:1" CRM_meta_notify_master_uname="controller-0" CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="controller-1 controller-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="controller-1" CRM_meta_notify_type="post" CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000"  wait_last_known_master="true"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="86" operation="notify" operation_key="redis-master_post_notify_stopped_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="133" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:0_post_notify_stonith_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="39">
60de42
+    <action_set>
60de42
+      <rsc_op id="194" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:2_pre_notify_stop_0" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
60de42
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="controller-1 controller-0 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2" CRM_meta_notify_available_uname="controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:3 redis:4 redis:5 redis:6 redis:7 redis:8" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource="redis:1" CRM_meta_notify_master_uname="controller-0" CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="controller-1 controller-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="controller-1" CRM_meta_notify_type="pre" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000"  wait_last_known_master="true"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="84" operation="notify" operation_key="redis-master_pre_notify_stop_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="40" priority="1000000">
60de42
+    <action_set>
60de42
+      <rsc_op id="136" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:2_post_notify_stop_0" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
60de42
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_active_resource="redis:0 redis:1 redis:2" CRM_meta_notify_active_uname="controller-1 controller-0 controller-2" CRM_meta_notify_all_uname="controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2" CRM_meta_notify_available_uname="controller-2 controller-1 controller-0" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="redis:3 redis:4 redis:5 redis:6 redis:7 redis:8" CRM_meta_notify_key_operation="stonith" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource="redis:1" CRM_meta_notify_master_uname="controller-0" CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource="redis:0 redis:2" CRM_meta_notify_slave_uname="controller-1 controller-2" CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="redis:0" CRM_meta_notify_stop_uname="controller-1" CRM_meta_notify_type="post" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000"  wait_last_known_master="true"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="86" operation="notify" operation_key="redis-master_post_notify_stopped_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="133" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:0_post_notify_stonith_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="41" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="87" operation="notified" operation_key="redis-master_confirmed-post_notify_stopped_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="86" operation="notify" operation_key="redis-master_post_notify_stopped_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="135" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:1_post_notify_stop_0" on_node="controller-0" on_node_uuid="1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="136" operation="notify" operation_key="redis_post_notify_stop_0" internal_operation_key="redis:2_post_notify_stop_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="42" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="86" operation="notify" operation_key="redis-master_post_notify_stopped_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="83" operation="stopped" operation_key="redis-master_stopped_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="85" operation="notified" operation_key="redis-master_confirmed-pre_notify_stop_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="43">
60de42
+    <action_set>
60de42
+      <pseudo_event id="85" operation="notified" operation_key="redis-master_confirmed-pre_notify_stop_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="84" operation="notify" operation_key="redis-master_pre_notify_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="193" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:1_pre_notify_stop_0" on_node="controller-0" on_node_uuid="1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="194" operation="notify" operation_key="redis_pre_notify_stop_0" internal_operation_key="redis:2_pre_notify_stop_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="44">
60de42
+    <action_set>
60de42
+      <pseudo_event id="84" operation="notify" operation_key="redis-master_pre_notify_stop_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs/>
60de42
+  </synapse>
60de42
+  <synapse id="45" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="83" operation="stopped" operation_key="redis-master_stopped_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="69" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="82" operation="stop" operation_key="redis-master_stop_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="46">
60de42
+    <action_set>
60de42
+      <pseudo_event id="82" operation="stop" operation_key="redis-master_stop_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="true" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="85" operation="notified" operation_key="redis-master_confirmed-pre_notify_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="47">
60de42
+    <action_set>
60de42
+      <rsc_op id="106" operation="monitor" operation_key="ip-172.17.1.14_monitor_10000" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="ip-172.17.1.14" class="ocf" provider="heartbeat" type="IPaddr2"/>
60de42
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" cidr_netmask="32"  ip="172.17.1.14"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="105" operation="start" operation_key="ip-172.17.1.14_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="48">
60de42
+    <action_set>
60de42
+      <rsc_op id="105" operation="start" operation_key="ip-172.17.1.14_start_0" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="ip-172.17.1.14" class="ocf" provider="heartbeat" type="IPaddr2"/>
60de42
+        <attributes CRM_meta_name="start" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" cidr_netmask="32"  ip="172.17.1.14"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="104" operation="stop" operation_key="ip-172.17.1.14_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="49">
60de42
+    <action_set>
60de42
+      <pseudo_event id="104" operation="stop" operation_key="ip-172.17.1.14_stop_0">
60de42
+        <attributes CRM_meta_name="stop" CRM_meta_timeout="20000" cidr_netmask="32"  ip="172.17.1.14"/>
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="123" operation="stopped" operation_key="haproxy-clone_stopped_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="50">
60de42
+    <action_set>
60de42
+      <rsc_op id="109" operation="monitor" operation_key="ip-172.17.1.17_monitor_10000" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="ip-172.17.1.17" class="ocf" provider="heartbeat" type="IPaddr2"/>
60de42
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" cidr_netmask="32"  ip="172.17.1.17"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="108" operation="start" operation_key="ip-172.17.1.17_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="51">
60de42
+    <action_set>
60de42
+      <rsc_op id="108" operation="start" operation_key="ip-172.17.1.17_start_0" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="ip-172.17.1.17" class="ocf" provider="heartbeat" type="IPaddr2"/>
60de42
+        <attributes CRM_meta_name="start" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" cidr_netmask="32"  ip="172.17.1.17"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="107" operation="stop" operation_key="ip-172.17.1.17_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="52">
60de42
+    <action_set>
60de42
+      <pseudo_event id="107" operation="stop" operation_key="ip-172.17.1.17_stop_0">
60de42
+        <attributes CRM_meta_name="stop" CRM_meta_timeout="20000" cidr_netmask="32"  ip="172.17.1.17"/>
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="123" operation="stopped" operation_key="haproxy-clone_stopped_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="53">
60de42
+    <action_set>
60de42
+      <rsc_op id="114" operation="monitor" operation_key="ip-172.17.4.11_monitor_10000" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="ip-172.17.4.11" class="ocf" provider="heartbeat" type="IPaddr2"/>
60de42
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" cidr_netmask="32"  ip="172.17.4.11"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="113" operation="start" operation_key="ip-172.17.4.11_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="54">
60de42
+    <action_set>
60de42
+      <rsc_op id="113" operation="start" operation_key="ip-172.17.4.11_start_0" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="ip-172.17.4.11" class="ocf" provider="heartbeat" type="IPaddr2"/>
60de42
+        <attributes CRM_meta_name="start" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" cidr_netmask="32"  ip="172.17.4.11"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="112" operation="stop" operation_key="ip-172.17.4.11_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="55">
60de42
+    <action_set>
60de42
+      <pseudo_event id="112" operation="stop" operation_key="ip-172.17.4.11_stop_0">
60de42
+        <attributes CRM_meta_name="stop" CRM_meta_timeout="20000" cidr_netmask="32"  ip="172.17.4.11"/>
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="123" operation="stopped" operation_key="haproxy-clone_stopped_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="56">
60de42
+    <action_set>
60de42
+      <pseudo_event id="115" operation="stop" operation_key="haproxy_stop_0" internal_operation_key="haproxy:0_stop_0">
60de42
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_timeout="200000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="122" operation="stop" operation_key="haproxy-clone_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="57" priority="1000000">
60de42
+    <action_set>
60de42
+      <pseudo_event id="123" operation="stopped" operation_key="haproxy-clone_stopped_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="115" operation="stop" operation_key="haproxy_stop_0" internal_operation_key="haproxy:0_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="122" operation="stop" operation_key="haproxy-clone_stop_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="58">
60de42
+    <action_set>
60de42
+      <pseudo_event id="122" operation="stop" operation_key="haproxy-clone_stop_0">
60de42
+        <attributes CRM_meta_clone_max="9" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="59">
60de42
+    <action_set>
60de42
+      <rsc_op id="126" operation="stop" operation_key="stonith-fence_ipmilan-525400bbf613_stop_0" on_node="controller-0" on_node_uuid="1">
60de42
+        <primitive id="stonith-fence_ipmilan-525400bbf613" class="stonith" type="fence_ipmilan"/>
60de42
+        <attributes CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" action="reboot"  ipaddr="172.16.0.1" ipport="6237" lanplus="true" login="admin" passwd="****" pcmk_host_list="controller-2" privlvl="administrator"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs/>
60de42
+  </synapse>
60de42
+  <synapse id="60">
60de42
+    <action_set>
60de42
+      <rsc_op id="22" operation="start" operation_key="stonith-fence_ipmilan-525400bbf613_start_0" on_node="controller-0" on_node_uuid="1">
60de42
+        <primitive id="stonith-fence_ipmilan-525400bbf613" class="stonith" type="fence_ipmilan"/>
60de42
+        <attributes CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" action="reboot"  ipaddr="172.16.0.1" ipport="6237" lanplus="true" login="admin" passwd="****" pcmk_host_list="controller-2" privlvl="administrator"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="126" operation="stop" operation_key="stonith-fence_ipmilan-525400bbf613_stop_0" on_node="controller-0" on_node_uuid="1"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="61">
60de42
+    <action_set>
60de42
+      <rsc_op id="6" operation="monitor" operation_key="stonith-fence_ipmilan-525400bbf613_monitor_60000" on_node="controller-0" on_node_uuid="1">
60de42
+        <primitive id="stonith-fence_ipmilan-525400bbf613" class="stonith" type="fence_ipmilan"/>
60de42
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" action="reboot"  ipaddr="172.16.0.1" ipport="6237" lanplus="true" login="admin" passwd="****" pcmk_host_list="controller-2" privlvl="administrator"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="22" operation="start" operation_key="stonith-fence_ipmilan-525400bbf613_start_0" on_node="controller-0" on_node_uuid="1"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="62">
60de42
+    <action_set>
60de42
+      <rsc_op id="127" operation="stop" operation_key="stonith-fence_ipmilan-525400b4f6bd_stop_0" on_node="controller-0" on_node_uuid="1">
60de42
+        <primitive id="stonith-fence_ipmilan-525400b4f6bd" class="stonith" type="fence_ipmilan"/>
60de42
+        <attributes CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" action="reboot"  ipaddr="172.16.0.1" ipport="6236" lanplus="true" login="admin" passwd="****" pcmk_host_list="controller-1" privlvl="administrator"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs/>
60de42
+  </synapse>
60de42
+  <synapse id="63">
60de42
+    <action_set>
60de42
+      <rsc_op id="23" operation="start" operation_key="stonith-fence_ipmilan-525400b4f6bd_start_0" on_node="controller-0" on_node_uuid="1">
60de42
+        <primitive id="stonith-fence_ipmilan-525400b4f6bd" class="stonith" type="fence_ipmilan"/>
60de42
+        <attributes CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" action="reboot"  ipaddr="172.16.0.1" ipport="6236" lanplus="true" login="admin" passwd="****" pcmk_host_list="controller-1" privlvl="administrator"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="127" operation="stop" operation_key="stonith-fence_ipmilan-525400b4f6bd_stop_0" on_node="controller-0" on_node_uuid="1"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="64">
60de42
+    <action_set>
60de42
+      <rsc_op id="11" operation="monitor" operation_key="stonith-fence_ipmilan-525400b4f6bd_monitor_60000" on_node="controller-0" on_node_uuid="1">
60de42
+        <primitive id="stonith-fence_ipmilan-525400b4f6bd" class="stonith" type="fence_ipmilan"/>
60de42
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="controller-0" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" action="reboot"  ipaddr="172.16.0.1" ipport="6236" lanplus="true" login="admin" passwd="****" pcmk_host_list="controller-1" privlvl="administrator"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="23" operation="start" operation_key="stonith-fence_ipmilan-525400b4f6bd_start_0" on_node="controller-0" on_node_uuid="1"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="65">
60de42
+    <action_set>
60de42
+      <rsc_op id="130" operation="monitor" operation_key="stonith-fence_ipmilan-5254005bdbb5_monitor_60000" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="stonith-fence_ipmilan-5254005bdbb5" class="stonith" type="fence_ipmilan"/>
60de42
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" action="reboot"  ipaddr="172.16.0.1" ipport="6235" lanplus="true" login="admin" passwd="****" pcmk_host_list="controller-0" privlvl="administrator"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <rsc_op id="129" operation="start" operation_key="stonith-fence_ipmilan-5254005bdbb5_start_0" on_node="controller-2" on_node_uuid="3"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="66">
60de42
+    <action_set>
60de42
+      <rsc_op id="129" operation="start" operation_key="stonith-fence_ipmilan-5254005bdbb5_start_0" on_node="controller-2" on_node_uuid="3">
60de42
+        <primitive id="stonith-fence_ipmilan-5254005bdbb5" class="stonith" type="fence_ipmilan"/>
60de42
+        <attributes CRM_meta_on_node="controller-2" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" action="reboot"  ipaddr="172.16.0.1" ipport="6235" lanplus="true" login="admin" passwd="****" pcmk_host_list="controller-0" privlvl="administrator"/>
60de42
+      </rsc_op>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="128" operation="stop" operation_key="stonith-fence_ipmilan-5254005bdbb5_stop_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="67">
60de42
+    <action_set>
60de42
+      <pseudo_event id="128" operation="stop" operation_key="stonith-fence_ipmilan-5254005bdbb5_stop_0">
60de42
+        <attributes CRM_meta_timeout="20000" action="reboot"  ipaddr="172.16.0.1" ipport="6235" lanplus="true" login="admin" passwd="****" pcmk_host_list="controller-0" privlvl="administrator"/>
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs/>
60de42
+  </synapse>
60de42
+  <synapse id="68">
60de42
+    <action_set>
60de42
+      <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2">
60de42
+        <attributes CRM_meta_cinder_volume_role="true" CRM_meta_haproxy_role="true" CRM_meta_on_node="controller-1" CRM_meta_on_node_uuid="2" CRM_meta_redis_role="true" CRM_meta_stonith_action="reboot" />
60de42
+        <downed>
60de42
+          <node id="2"/>
60de42
+        </downed>
60de42
+      </crm_event>
60de42
+    </action_set>
60de42
+    <inputs/>
60de42
+  </synapse>
60de42
+  <synapse id="69">
60de42
+    <action_set>
60de42
+      <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete">
60de42
+        <attributes />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <crm_event id="132" operation="stonith" operation_key="stonith-controller-1-reboot" on_node="controller-1" on_node_uuid="2"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+  <synapse id="70">
60de42
+    <action_set>
60de42
+      <pseudo_event id="21" operation="all_stopped" operation_key="all_stopped">
60de42
+        <attributes />
60de42
+      </pseudo_event>
60de42
+    </action_set>
60de42
+    <inputs>
60de42
+      <trigger>
60de42
+        <pseudo_event id="26" operation="stop" operation_key="messaging-1_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="31" operation="stop" operation_key="galera-0_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="36" operation="stop" operation_key="galera-2_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="43" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:2_stop_0" on_node="messaging-1" on_node_uuid="messaging-1" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="55" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:1_stop_0" on_node="galera-2" on_node_uuid="galera-2" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="59" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:2_stop_0" on_node="galera-0" on_node_uuid="galera-0" router_node="controller-1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="69" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="87" operation="notified" operation_key="redis-master_confirmed-post_notify_stopped_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="104" operation="stop" operation_key="ip-172.17.1.14_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="107" operation="stop" operation_key="ip-172.17.1.17_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="112" operation="stop" operation_key="ip-172.17.4.11_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="115" operation="stop" operation_key="haproxy_stop_0" internal_operation_key="haproxy:0_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="126" operation="stop" operation_key="stonith-fence_ipmilan-525400bbf613_stop_0" on_node="controller-0" on_node_uuid="1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <rsc_op id="127" operation="stop" operation_key="stonith-fence_ipmilan-525400b4f6bd_stop_0" on_node="controller-0" on_node_uuid="1"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="128" operation="stop" operation_key="stonith-fence_ipmilan-5254005bdbb5_stop_0"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="131" operation="stonith_complete" operation_key="stonith_complete"/>
60de42
+      </trigger>
60de42
+      <trigger>
60de42
+        <pseudo_event id="134" operation="notified" operation_key="redis_notified_0" internal_operation_key="redis:0_confirmed-post_notify_stonith_0"/>
60de42
+      </trigger>
60de42
+    </inputs>
60de42
+  </synapse>
60de42
+</transition_graph>
60de42
diff --git a/pengine/test10/remote-recovery.scores b/pengine/test10/remote-recovery.scores
60de42
new file mode 100644
60de42
index 0000000..56a18eb
60de42
--- /dev/null
60de42
+++ b/pengine/test10/remote-recovery.scores
60de42
@@ -0,0 +1,848 @@
60de42
+Allocation scores:
60de42
+Using the original execution date of: 2017-05-03 13:33:24Z
60de42
+clone_color: galera-master allocation score on controller-0: -INFINITY
60de42
+clone_color: galera-master allocation score on controller-1: -INFINITY
60de42
+clone_color: galera-master allocation score on controller-2: -INFINITY
60de42
+clone_color: galera-master allocation score on galera-0: 0
60de42
+clone_color: galera-master allocation score on galera-1: 0
60de42
+clone_color: galera-master allocation score on galera-2: 0
60de42
+clone_color: galera-master allocation score on messaging-0: -INFINITY
60de42
+clone_color: galera-master allocation score on messaging-1: -INFINITY
60de42
+clone_color: galera-master allocation score on messaging-2: -INFINITY
60de42
+clone_color: galera:0 allocation score on controller-0: -INFINITY
60de42
+clone_color: galera:0 allocation score on controller-1: -INFINITY
60de42
+clone_color: galera:0 allocation score on controller-2: -INFINITY
60de42
+clone_color: galera:0 allocation score on galera-0: 0
60de42
+clone_color: galera:0 allocation score on galera-1: INFINITY
60de42
+clone_color: galera:0 allocation score on galera-2: 0
60de42
+clone_color: galera:0 allocation score on messaging-0: -INFINITY
60de42
+clone_color: galera:0 allocation score on messaging-1: -INFINITY
60de42
+clone_color: galera:0 allocation score on messaging-2: -INFINITY
60de42
+clone_color: galera:1 allocation score on controller-0: -INFINITY
60de42
+clone_color: galera:1 allocation score on controller-1: -INFINITY
60de42
+clone_color: galera:1 allocation score on controller-2: -INFINITY
60de42
+clone_color: galera:1 allocation score on galera-0: 0
60de42
+clone_color: galera:1 allocation score on galera-1: 0
60de42
+clone_color: galera:1 allocation score on galera-2: INFINITY
60de42
+clone_color: galera:1 allocation score on messaging-0: -INFINITY
60de42
+clone_color: galera:1 allocation score on messaging-1: -INFINITY
60de42
+clone_color: galera:1 allocation score on messaging-2: -INFINITY
60de42
+clone_color: galera:2 allocation score on controller-0: -INFINITY
60de42
+clone_color: galera:2 allocation score on controller-1: -INFINITY
60de42
+clone_color: galera:2 allocation score on controller-2: -INFINITY
60de42
+clone_color: galera:2 allocation score on galera-0: INFINITY
60de42
+clone_color: galera:2 allocation score on galera-1: 0
60de42
+clone_color: galera:2 allocation score on galera-2: 0
60de42
+clone_color: galera:2 allocation score on messaging-0: -INFINITY
60de42
+clone_color: galera:2 allocation score on messaging-1: -INFINITY
60de42
+clone_color: galera:2 allocation score on messaging-2: -INFINITY
60de42
+clone_color: galera:3 allocation score on controller-0: -INFINITY
60de42
+clone_color: galera:3 allocation score on controller-1: -INFINITY
60de42
+clone_color: galera:3 allocation score on controller-2: -INFINITY
60de42
+clone_color: galera:3 allocation score on galera-0: 0
60de42
+clone_color: galera:3 allocation score on galera-1: 0
60de42
+clone_color: galera:3 allocation score on galera-2: 0
60de42
+clone_color: galera:3 allocation score on messaging-0: -INFINITY
60de42
+clone_color: galera:3 allocation score on messaging-1: -INFINITY
60de42
+clone_color: galera:3 allocation score on messaging-2: -INFINITY
60de42
+clone_color: galera:4 allocation score on controller-0: -INFINITY
60de42
+clone_color: galera:4 allocation score on controller-1: -INFINITY
60de42
+clone_color: galera:4 allocation score on controller-2: -INFINITY
60de42
+clone_color: galera:4 allocation score on galera-0: 0
60de42
+clone_color: galera:4 allocation score on galera-1: 0
60de42
+clone_color: galera:4 allocation score on galera-2: 0
60de42
+clone_color: galera:4 allocation score on messaging-0: -INFINITY
60de42
+clone_color: galera:4 allocation score on messaging-1: -INFINITY
60de42
+clone_color: galera:4 allocation score on messaging-2: -INFINITY
60de42
+clone_color: galera:5 allocation score on controller-0: -INFINITY
60de42
+clone_color: galera:5 allocation score on controller-1: -INFINITY
60de42
+clone_color: galera:5 allocation score on controller-2: -INFINITY
60de42
+clone_color: galera:5 allocation score on galera-0: 0
60de42
+clone_color: galera:5 allocation score on galera-1: 0
60de42
+clone_color: galera:5 allocation score on galera-2: 0
60de42
+clone_color: galera:5 allocation score on messaging-0: -INFINITY
60de42
+clone_color: galera:5 allocation score on messaging-1: -INFINITY
60de42
+clone_color: galera:5 allocation score on messaging-2: -INFINITY
60de42
+clone_color: galera:6 allocation score on controller-0: -INFINITY
60de42
+clone_color: galera:6 allocation score on controller-1: -INFINITY
60de42
+clone_color: galera:6 allocation score on controller-2: -INFINITY
60de42
+clone_color: galera:6 allocation score on galera-0: 0
60de42
+clone_color: galera:6 allocation score on galera-1: 0
60de42
+clone_color: galera:6 allocation score on galera-2: 0
60de42
+clone_color: galera:6 allocation score on messaging-0: -INFINITY
60de42
+clone_color: galera:6 allocation score on messaging-1: -INFINITY
60de42
+clone_color: galera:6 allocation score on messaging-2: -INFINITY
60de42
+clone_color: galera:7 allocation score on controller-0: -INFINITY
60de42
+clone_color: galera:7 allocation score on controller-1: -INFINITY
60de42
+clone_color: galera:7 allocation score on controller-2: -INFINITY
60de42
+clone_color: galera:7 allocation score on galera-0: 0
60de42
+clone_color: galera:7 allocation score on galera-1: 0
60de42
+clone_color: galera:7 allocation score on galera-2: 0
60de42
+clone_color: galera:7 allocation score on messaging-0: -INFINITY
60de42
+clone_color: galera:7 allocation score on messaging-1: -INFINITY
60de42
+clone_color: galera:7 allocation score on messaging-2: -INFINITY
60de42
+clone_color: galera:8 allocation score on controller-0: -INFINITY
60de42
+clone_color: galera:8 allocation score on controller-1: -INFINITY
60de42
+clone_color: galera:8 allocation score on controller-2: -INFINITY
60de42
+clone_color: galera:8 allocation score on galera-0: 0
60de42
+clone_color: galera:8 allocation score on galera-1: 0
60de42
+clone_color: galera:8 allocation score on galera-2: 0
60de42
+clone_color: galera:8 allocation score on messaging-0: -INFINITY
60de42
+clone_color: galera:8 allocation score on messaging-1: -INFINITY
60de42
+clone_color: galera:8 allocation score on messaging-2: -INFINITY
60de42
+clone_color: haproxy-clone allocation score on controller-0: INFINITY
60de42
+clone_color: haproxy-clone allocation score on controller-1: 0
60de42
+clone_color: haproxy-clone allocation score on controller-2: 0
60de42
+clone_color: haproxy-clone allocation score on galera-0: -INFINITY
60de42
+clone_color: haproxy-clone allocation score on galera-1: -INFINITY
60de42
+clone_color: haproxy-clone allocation score on galera-2: -INFINITY
60de42
+clone_color: haproxy-clone allocation score on messaging-0: -INFINITY
60de42
+clone_color: haproxy-clone allocation score on messaging-1: -INFINITY
60de42
+clone_color: haproxy-clone allocation score on messaging-2: -INFINITY
60de42
+clone_color: haproxy:0 allocation score on controller-0: 0
60de42
+clone_color: haproxy:0 allocation score on controller-1: INFINITY
60de42
+clone_color: haproxy:0 allocation score on controller-2: 0
60de42
+clone_color: haproxy:0 allocation score on galera-0: -INFINITY
60de42
+clone_color: haproxy:0 allocation score on galera-1: -INFINITY
60de42
+clone_color: haproxy:0 allocation score on galera-2: -INFINITY
60de42
+clone_color: haproxy:0 allocation score on messaging-0: -INFINITY
60de42
+clone_color: haproxy:0 allocation score on messaging-1: -INFINITY
60de42
+clone_color: haproxy:0 allocation score on messaging-2: -INFINITY
60de42
+clone_color: haproxy:1 allocation score on controller-0: INFINITY
60de42
+clone_color: haproxy:1 allocation score on controller-1: 0
60de42
+clone_color: haproxy:1 allocation score on controller-2: 0
60de42
+clone_color: haproxy:1 allocation score on galera-0: -INFINITY
60de42
+clone_color: haproxy:1 allocation score on galera-1: -INFINITY
60de42
+clone_color: haproxy:1 allocation score on galera-2: -INFINITY
60de42
+clone_color: haproxy:1 allocation score on messaging-0: -INFINITY
60de42
+clone_color: haproxy:1 allocation score on messaging-1: -INFINITY
60de42
+clone_color: haproxy:1 allocation score on messaging-2: -INFINITY
60de42
+clone_color: haproxy:2 allocation score on controller-0: 0
60de42
+clone_color: haproxy:2 allocation score on controller-1: 0
60de42
+clone_color: haproxy:2 allocation score on controller-2: INFINITY
60de42
+clone_color: haproxy:2 allocation score on galera-0: -INFINITY
60de42
+clone_color: haproxy:2 allocation score on galera-1: -INFINITY
60de42
+clone_color: haproxy:2 allocation score on galera-2: -INFINITY
60de42
+clone_color: haproxy:2 allocation score on messaging-0: -INFINITY
60de42
+clone_color: haproxy:2 allocation score on messaging-1: -INFINITY
60de42
+clone_color: haproxy:2 allocation score on messaging-2: -INFINITY
60de42
+clone_color: haproxy:3 allocation score on controller-0: 0
60de42
+clone_color: haproxy:3 allocation score on controller-1: 0
60de42
+clone_color: haproxy:3 allocation score on controller-2: 0
60de42
+clone_color: haproxy:3 allocation score on galera-0: -INFINITY
60de42
+clone_color: haproxy:3 allocation score on galera-1: -INFINITY
60de42
+clone_color: haproxy:3 allocation score on galera-2: -INFINITY
60de42
+clone_color: haproxy:3 allocation score on messaging-0: -INFINITY
60de42
+clone_color: haproxy:3 allocation score on messaging-1: -INFINITY
60de42
+clone_color: haproxy:3 allocation score on messaging-2: -INFINITY
60de42
+clone_color: haproxy:4 allocation score on controller-0: 0
60de42
+clone_color: haproxy:4 allocation score on controller-1: 0
60de42
+clone_color: haproxy:4 allocation score on controller-2: 0
60de42
+clone_color: haproxy:4 allocation score on galera-0: -INFINITY
60de42
+clone_color: haproxy:4 allocation score on galera-1: -INFINITY
60de42
+clone_color: haproxy:4 allocation score on galera-2: -INFINITY
60de42
+clone_color: haproxy:4 allocation score on messaging-0: -INFINITY
60de42
+clone_color: haproxy:4 allocation score on messaging-1: -INFINITY
60de42
+clone_color: haproxy:4 allocation score on messaging-2: -INFINITY
60de42
+clone_color: haproxy:5 allocation score on controller-0: 0
60de42
+clone_color: haproxy:5 allocation score on controller-1: 0
60de42
+clone_color: haproxy:5 allocation score on controller-2: 0
60de42
+clone_color: haproxy:5 allocation score on galera-0: -INFINITY
60de42
+clone_color: haproxy:5 allocation score on galera-1: -INFINITY
60de42
+clone_color: haproxy:5 allocation score on galera-2: -INFINITY
60de42
+clone_color: haproxy:5 allocation score on messaging-0: -INFINITY
60de42
+clone_color: haproxy:5 allocation score on messaging-1: -INFINITY
60de42
+clone_color: haproxy:5 allocation score on messaging-2: -INFINITY
60de42
+clone_color: haproxy:6 allocation score on controller-0: 0
60de42
+clone_color: haproxy:6 allocation score on controller-1: 0
60de42
+clone_color: haproxy:6 allocation score on controller-2: 0
60de42
+clone_color: haproxy:6 allocation score on galera-0: -INFINITY
60de42
+clone_color: haproxy:6 allocation score on galera-1: -INFINITY
60de42
+clone_color: haproxy:6 allocation score on galera-2: -INFINITY
60de42
+clone_color: haproxy:6 allocation score on messaging-0: -INFINITY
60de42
+clone_color: haproxy:6 allocation score on messaging-1: -INFINITY
60de42
+clone_color: haproxy:6 allocation score on messaging-2: -INFINITY
60de42
+clone_color: haproxy:7 allocation score on controller-0: 0
60de42
+clone_color: haproxy:7 allocation score on controller-1: 0
60de42
+clone_color: haproxy:7 allocation score on controller-2: 0
60de42
+clone_color: haproxy:7 allocation score on galera-0: -INFINITY
60de42
+clone_color: haproxy:7 allocation score on galera-1: -INFINITY
60de42
+clone_color: haproxy:7 allocation score on galera-2: -INFINITY
60de42
+clone_color: haproxy:7 allocation score on messaging-0: -INFINITY
60de42
+clone_color: haproxy:7 allocation score on messaging-1: -INFINITY
60de42
+clone_color: haproxy:7 allocation score on messaging-2: -INFINITY
60de42
+clone_color: haproxy:8 allocation score on controller-0: 0
60de42
+clone_color: haproxy:8 allocation score on controller-1: 0
60de42
+clone_color: haproxy:8 allocation score on controller-2: 0
60de42
+clone_color: haproxy:8 allocation score on galera-0: -INFINITY
60de42
+clone_color: haproxy:8 allocation score on galera-1: -INFINITY
60de42
+clone_color: haproxy:8 allocation score on galera-2: -INFINITY
60de42
+clone_color: haproxy:8 allocation score on messaging-0: -INFINITY
60de42
+clone_color: haproxy:8 allocation score on messaging-1: -INFINITY
60de42
+clone_color: haproxy:8 allocation score on messaging-2: -INFINITY
60de42
+clone_color: rabbitmq-clone allocation score on controller-0: -INFINITY
60de42
+clone_color: rabbitmq-clone allocation score on controller-1: -INFINITY
60de42
+clone_color: rabbitmq-clone allocation score on controller-2: -INFINITY
60de42
+clone_color: rabbitmq-clone allocation score on galera-0: -INFINITY
60de42
+clone_color: rabbitmq-clone allocation score on galera-1: -INFINITY
60de42
+clone_color: rabbitmq-clone allocation score on galera-2: -INFINITY
60de42
+clone_color: rabbitmq-clone allocation score on messaging-0: 0
60de42
+clone_color: rabbitmq-clone allocation score on messaging-1: 0
60de42
+clone_color: rabbitmq-clone allocation score on messaging-2: 0
60de42
+clone_color: rabbitmq:0 allocation score on controller-0: -INFINITY
60de42
+clone_color: rabbitmq:0 allocation score on controller-1: -INFINITY
60de42
+clone_color: rabbitmq:0 allocation score on controller-2: -INFINITY
60de42
+clone_color: rabbitmq:0 allocation score on galera-0: -INFINITY
60de42
+clone_color: rabbitmq:0 allocation score on galera-1: -INFINITY
60de42
+clone_color: rabbitmq:0 allocation score on galera-2: -INFINITY
60de42
+clone_color: rabbitmq:0 allocation score on messaging-0: 0
60de42
+clone_color: rabbitmq:0 allocation score on messaging-1: 0
60de42
+clone_color: rabbitmq:0 allocation score on messaging-2: INFINITY
60de42
+clone_color: rabbitmq:1 allocation score on controller-0: -INFINITY
60de42
+clone_color: rabbitmq:1 allocation score on controller-1: -INFINITY
60de42
+clone_color: rabbitmq:1 allocation score on controller-2: -INFINITY
60de42
+clone_color: rabbitmq:1 allocation score on galera-0: -INFINITY
60de42
+clone_color: rabbitmq:1 allocation score on galera-1: -INFINITY
60de42
+clone_color: rabbitmq:1 allocation score on galera-2: -INFINITY
60de42
+clone_color: rabbitmq:1 allocation score on messaging-0: INFINITY
60de42
+clone_color: rabbitmq:1 allocation score on messaging-1: 0
60de42
+clone_color: rabbitmq:1 allocation score on messaging-2: 0
60de42
+clone_color: rabbitmq:2 allocation score on controller-0: -INFINITY
60de42
+clone_color: rabbitmq:2 allocation score on controller-1: -INFINITY
60de42
+clone_color: rabbitmq:2 allocation score on controller-2: -INFINITY
60de42
+clone_color: rabbitmq:2 allocation score on galera-0: -INFINITY
60de42
+clone_color: rabbitmq:2 allocation score on galera-1: -INFINITY
60de42
+clone_color: rabbitmq:2 allocation score on galera-2: -INFINITY
60de42
+clone_color: rabbitmq:2 allocation score on messaging-0: 0
60de42
+clone_color: rabbitmq:2 allocation score on messaging-1: INFINITY
60de42
+clone_color: rabbitmq:2 allocation score on messaging-2: 0
60de42
+clone_color: rabbitmq:3 allocation score on controller-0: -INFINITY
60de42
+clone_color: rabbitmq:3 allocation score on controller-1: -INFINITY
60de42
+clone_color: rabbitmq:3 allocation score on controller-2: -INFINITY
60de42
+clone_color: rabbitmq:3 allocation score on galera-0: -INFINITY
60de42
+clone_color: rabbitmq:3 allocation score on galera-1: -INFINITY
60de42
+clone_color: rabbitmq:3 allocation score on galera-2: -INFINITY
60de42
+clone_color: rabbitmq:3 allocation score on messaging-0: 0
60de42
+clone_color: rabbitmq:3 allocation score on messaging-1: 0
60de42
+clone_color: rabbitmq:3 allocation score on messaging-2: 0
60de42
+clone_color: rabbitmq:4 allocation score on controller-0: -INFINITY
60de42
+clone_color: rabbitmq:4 allocation score on controller-1: -INFINITY
60de42
+clone_color: rabbitmq:4 allocation score on controller-2: -INFINITY
60de42
+clone_color: rabbitmq:4 allocation score on galera-0: -INFINITY
60de42
+clone_color: rabbitmq:4 allocation score on galera-1: -INFINITY
60de42
+clone_color: rabbitmq:4 allocation score on galera-2: -INFINITY
60de42
+clone_color: rabbitmq:4 allocation score on messaging-0: 0
60de42
+clone_color: rabbitmq:4 allocation score on messaging-1: 0
60de42
+clone_color: rabbitmq:4 allocation score on messaging-2: 0
60de42
+clone_color: rabbitmq:5 allocation score on controller-0: -INFINITY
60de42
+clone_color: rabbitmq:5 allocation score on controller-1: -INFINITY
60de42
+clone_color: rabbitmq:5 allocation score on controller-2: -INFINITY
60de42
+clone_color: rabbitmq:5 allocation score on galera-0: -INFINITY
60de42
+clone_color: rabbitmq:5 allocation score on galera-1: -INFINITY
60de42
+clone_color: rabbitmq:5 allocation score on galera-2: -INFINITY
60de42
+clone_color: rabbitmq:5 allocation score on messaging-0: 0
60de42
+clone_color: rabbitmq:5 allocation score on messaging-1: 0
60de42
+clone_color: rabbitmq:5 allocation score on messaging-2: 0
60de42
+clone_color: rabbitmq:6 allocation score on controller-0: -INFINITY
60de42
+clone_color: rabbitmq:6 allocation score on controller-1: -INFINITY
60de42
+clone_color: rabbitmq:6 allocation score on controller-2: -INFINITY
60de42
+clone_color: rabbitmq:6 allocation score on galera-0: -INFINITY
60de42
+clone_color: rabbitmq:6 allocation score on galera-1: -INFINITY
60de42
+clone_color: rabbitmq:6 allocation score on galera-2: -INFINITY
60de42
+clone_color: rabbitmq:6 allocation score on messaging-0: 0
60de42
+clone_color: rabbitmq:6 allocation score on messaging-1: 0
60de42
+clone_color: rabbitmq:6 allocation score on messaging-2: 0
60de42
+clone_color: rabbitmq:7 allocation score on controller-0: -INFINITY
60de42
+clone_color: rabbitmq:7 allocation score on controller-1: -INFINITY
60de42
+clone_color: rabbitmq:7 allocation score on controller-2: -INFINITY
60de42
+clone_color: rabbitmq:7 allocation score on galera-0: -INFINITY
60de42
+clone_color: rabbitmq:7 allocation score on galera-1: -INFINITY
60de42
+clone_color: rabbitmq:7 allocation score on galera-2: -INFINITY
60de42
+clone_color: rabbitmq:7 allocation score on messaging-0: 0
60de42
+clone_color: rabbitmq:7 allocation score on messaging-1: 0
60de42
+clone_color: rabbitmq:7 allocation score on messaging-2: 0
60de42
+clone_color: rabbitmq:8 allocation score on controller-0: -INFINITY
60de42
+clone_color: rabbitmq:8 allocation score on controller-1: -INFINITY
60de42
+clone_color: rabbitmq:8 allocation score on controller-2: -INFINITY
60de42
+clone_color: rabbitmq:8 allocation score on galera-0: -INFINITY
60de42
+clone_color: rabbitmq:8 allocation score on galera-1: -INFINITY
60de42
+clone_color: rabbitmq:8 allocation score on galera-2: -INFINITY
60de42
+clone_color: rabbitmq:8 allocation score on messaging-0: 0
60de42
+clone_color: rabbitmq:8 allocation score on messaging-1: 0
60de42
+clone_color: rabbitmq:8 allocation score on messaging-2: 0
60de42
+clone_color: redis-master allocation score on controller-0: 0
60de42
+clone_color: redis-master allocation score on controller-1: 0
60de42
+clone_color: redis-master allocation score on controller-2: 0
60de42
+clone_color: redis-master allocation score on galera-0: -INFINITY
60de42
+clone_color: redis-master allocation score on galera-1: -INFINITY
60de42
+clone_color: redis-master allocation score on galera-2: -INFINITY
60de42
+clone_color: redis-master allocation score on messaging-0: -INFINITY
60de42
+clone_color: redis-master allocation score on messaging-1: -INFINITY
60de42
+clone_color: redis-master allocation score on messaging-2: -INFINITY
60de42
+clone_color: redis:0 allocation score on controller-0: 0
60de42
+clone_color: redis:0 allocation score on controller-1: INFINITY
60de42
+clone_color: redis:0 allocation score on controller-2: 0
60de42
+clone_color: redis:0 allocation score on galera-0: -INFINITY
60de42
+clone_color: redis:0 allocation score on galera-1: -INFINITY
60de42
+clone_color: redis:0 allocation score on galera-2: -INFINITY
60de42
+clone_color: redis:0 allocation score on messaging-0: -INFINITY
60de42
+clone_color: redis:0 allocation score on messaging-1: -INFINITY
60de42
+clone_color: redis:0 allocation score on messaging-2: -INFINITY
60de42
+clone_color: redis:1 allocation score on controller-0: INFINITY
60de42
+clone_color: redis:1 allocation score on controller-1: 0
60de42
+clone_color: redis:1 allocation score on controller-2: 0
60de42
+clone_color: redis:1 allocation score on galera-0: -INFINITY
60de42
+clone_color: redis:1 allocation score on galera-1: -INFINITY
60de42
+clone_color: redis:1 allocation score on galera-2: -INFINITY
60de42
+clone_color: redis:1 allocation score on messaging-0: -INFINITY
60de42
+clone_color: redis:1 allocation score on messaging-1: -INFINITY
60de42
+clone_color: redis:1 allocation score on messaging-2: -INFINITY
60de42
+clone_color: redis:2 allocation score on controller-0: 0
60de42
+clone_color: redis:2 allocation score on controller-1: 0
60de42
+clone_color: redis:2 allocation score on controller-2: INFINITY
60de42
+clone_color: redis:2 allocation score on galera-0: -INFINITY
60de42
+clone_color: redis:2 allocation score on galera-1: -INFINITY
60de42
+clone_color: redis:2 allocation score on galera-2: -INFINITY
60de42
+clone_color: redis:2 allocation score on messaging-0: -INFINITY
60de42
+clone_color: redis:2 allocation score on messaging-1: -INFINITY
60de42
+clone_color: redis:2 allocation score on messaging-2: -INFINITY
60de42
+clone_color: redis:3 allocation score on controller-0: 0
60de42
+clone_color: redis:3 allocation score on controller-1: 0
60de42
+clone_color: redis:3 allocation score on controller-2: 0
60de42
+clone_color: redis:3 allocation score on galera-0: -INFINITY
60de42
+clone_color: redis:3 allocation score on galera-1: -INFINITY
60de42
+clone_color: redis:3 allocation score on galera-2: -INFINITY
60de42
+clone_color: redis:3 allocation score on messaging-0: -INFINITY
60de42
+clone_color: redis:3 allocation score on messaging-1: -INFINITY
60de42
+clone_color: redis:3 allocation score on messaging-2: -INFINITY
60de42
+clone_color: redis:4 allocation score on controller-0: 0
60de42
+clone_color: redis:4 allocation score on controller-1: 0
60de42
+clone_color: redis:4 allocation score on controller-2: 0
60de42
+clone_color: redis:4 allocation score on galera-0: -INFINITY
60de42
+clone_color: redis:4 allocation score on galera-1: -INFINITY
60de42
+clone_color: redis:4 allocation score on galera-2: -INFINITY
60de42
+clone_color: redis:4 allocation score on messaging-0: -INFINITY
60de42
+clone_color: redis:4 allocation score on messaging-1: -INFINITY
60de42
+clone_color: redis:4 allocation score on messaging-2: -INFINITY
60de42
+clone_color: redis:5 allocation score on controller-0: 0
60de42
+clone_color: redis:5 allocation score on controller-1: 0
60de42
+clone_color: redis:5 allocation score on controller-2: 0
60de42
+clone_color: redis:5 allocation score on galera-0: -INFINITY
60de42
+clone_color: redis:5 allocation score on galera-1: -INFINITY
60de42
+clone_color: redis:5 allocation score on galera-2: -INFINITY
60de42
+clone_color: redis:5 allocation score on messaging-0: -INFINITY
60de42
+clone_color: redis:5 allocation score on messaging-1: -INFINITY
60de42
+clone_color: redis:5 allocation score on messaging-2: -INFINITY
60de42
+clone_color: redis:6 allocation score on controller-0: 0
60de42
+clone_color: redis:6 allocation score on controller-1: 0
60de42
+clone_color: redis:6 allocation score on controller-2: 0
60de42
+clone_color: redis:6 allocation score on galera-0: -INFINITY
60de42
+clone_color: redis:6 allocation score on galera-1: -INFINITY
60de42
+clone_color: redis:6 allocation score on galera-2: -INFINITY
60de42
+clone_color: redis:6 allocation score on messaging-0: -INFINITY
60de42
+clone_color: redis:6 allocation score on messaging-1: -INFINITY
60de42
+clone_color: redis:6 allocation score on messaging-2: -INFINITY
60de42
+clone_color: redis:7 allocation score on controller-0: 0
60de42
+clone_color: redis:7 allocation score on controller-1: 0
60de42
+clone_color: redis:7 allocation score on controller-2: 0
60de42
+clone_color: redis:7 allocation score on galera-0: -INFINITY
60de42
+clone_color: redis:7 allocation score on galera-1: -INFINITY
60de42
+clone_color: redis:7 allocation score on galera-2: -INFINITY
60de42
+clone_color: redis:7 allocation score on messaging-0: -INFINITY
60de42
+clone_color: redis:7 allocation score on messaging-1: -INFINITY
60de42
+clone_color: redis:7 allocation score on messaging-2: -INFINITY
60de42
+clone_color: redis:8 allocation score on controller-0: 0
60de42
+clone_color: redis:8 allocation score on controller-1: 0
60de42
+clone_color: redis:8 allocation score on controller-2: 0
60de42
+clone_color: redis:8 allocation score on galera-0: -INFINITY
60de42
+clone_color: redis:8 allocation score on galera-1: -INFINITY
60de42
+clone_color: redis:8 allocation score on galera-2: -INFINITY
60de42
+clone_color: redis:8 allocation score on messaging-0: -INFINITY
60de42
+clone_color: redis:8 allocation score on messaging-1: -INFINITY
60de42
+clone_color: redis:8 allocation score on messaging-2: -INFINITY
60de42
+galera:0 promotion score on galera-1: 100
60de42
+galera:1 promotion score on galera-2: 100
60de42
+galera:2 promotion score on galera-0: 100
60de42
+galera:3 promotion score on none: 0
60de42
+galera:4 promotion score on none: 0
60de42
+galera:5 promotion score on none: 0
60de42
+galera:6 promotion score on none: 0
60de42
+galera:7 promotion score on none: 0
60de42
+galera:8 promotion score on none: 0
60de42
+native_color: galera-0 allocation score on controller-0: 0
60de42
+native_color: galera-0 allocation score on controller-1: INFINITY
60de42
+native_color: galera-0 allocation score on controller-2: 0
60de42
+native_color: galera-0 allocation score on galera-0: -INFINITY
60de42
+native_color: galera-0 allocation score on galera-1: -INFINITY
60de42
+native_color: galera-0 allocation score on galera-2: -INFINITY
60de42
+native_color: galera-0 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera-0 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera-0 allocation score on messaging-2: -INFINITY
60de42
+native_color: galera-1 allocation score on controller-0: INFINITY
60de42
+native_color: galera-1 allocation score on controller-1: 0
60de42
+native_color: galera-1 allocation score on controller-2: 0
60de42
+native_color: galera-1 allocation score on galera-0: -INFINITY
60de42
+native_color: galera-1 allocation score on galera-1: -INFINITY
60de42
+native_color: galera-1 allocation score on galera-2: -INFINITY
60de42
+native_color: galera-1 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera-1 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera-1 allocation score on messaging-2: -INFINITY
60de42
+native_color: galera-2 allocation score on controller-0: 0
60de42
+native_color: galera-2 allocation score on controller-1: INFINITY
60de42
+native_color: galera-2 allocation score on controller-2: 0
60de42
+native_color: galera-2 allocation score on galera-0: -INFINITY
60de42
+native_color: galera-2 allocation score on galera-1: -INFINITY
60de42
+native_color: galera-2 allocation score on galera-2: -INFINITY
60de42
+native_color: galera-2 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera-2 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera-2 allocation score on messaging-2: -INFINITY
60de42
+native_color: galera:0 allocation score on controller-0: -INFINITY
60de42
+native_color: galera:0 allocation score on controller-1: -INFINITY
60de42
+native_color: galera:0 allocation score on controller-2: -INFINITY
60de42
+native_color: galera:0 allocation score on galera-0: 0
60de42
+native_color: galera:0 allocation score on galera-1: INFINITY
60de42
+native_color: galera:0 allocation score on galera-2: 0
60de42
+native_color: galera:0 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera:0 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera:0 allocation score on messaging-2: -INFINITY
60de42
+native_color: galera:1 allocation score on controller-0: -INFINITY
60de42
+native_color: galera:1 allocation score on controller-1: -INFINITY
60de42
+native_color: galera:1 allocation score on controller-2: -INFINITY
60de42
+native_color: galera:1 allocation score on galera-0: 0
60de42
+native_color: galera:1 allocation score on galera-1: -INFINITY
60de42
+native_color: galera:1 allocation score on galera-2: INFINITY
60de42
+native_color: galera:1 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera:1 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera:1 allocation score on messaging-2: -INFINITY
60de42
+native_color: galera:2 allocation score on controller-0: -INFINITY
60de42
+native_color: galera:2 allocation score on controller-1: -INFINITY
60de42
+native_color: galera:2 allocation score on controller-2: -INFINITY
60de42
+native_color: galera:2 allocation score on galera-0: INFINITY
60de42
+native_color: galera:2 allocation score on galera-1: -INFINITY
60de42
+native_color: galera:2 allocation score on galera-2: -INFINITY
60de42
+native_color: galera:2 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera:2 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera:2 allocation score on messaging-2: -INFINITY
60de42
+native_color: galera:3 allocation score on controller-0: -INFINITY
60de42
+native_color: galera:3 allocation score on controller-1: -INFINITY
60de42
+native_color: galera:3 allocation score on controller-2: -INFINITY
60de42
+native_color: galera:3 allocation score on galera-0: -INFINITY
60de42
+native_color: galera:3 allocation score on galera-1: -INFINITY
60de42
+native_color: galera:3 allocation score on galera-2: -INFINITY
60de42
+native_color: galera:3 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera:3 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera:3 allocation score on messaging-2: -INFINITY
60de42
+native_color: galera:4 allocation score on controller-0: -INFINITY
60de42
+native_color: galera:4 allocation score on controller-1: -INFINITY
60de42
+native_color: galera:4 allocation score on controller-2: -INFINITY
60de42
+native_color: galera:4 allocation score on galera-0: -INFINITY
60de42
+native_color: galera:4 allocation score on galera-1: -INFINITY
60de42
+native_color: galera:4 allocation score on galera-2: -INFINITY
60de42
+native_color: galera:4 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera:4 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera:4 allocation score on messaging-2: -INFINITY
60de42
+native_color: galera:5 allocation score on controller-0: -INFINITY
60de42
+native_color: galera:5 allocation score on controller-1: -INFINITY
60de42
+native_color: galera:5 allocation score on controller-2: -INFINITY
60de42
+native_color: galera:5 allocation score on galera-0: -INFINITY
60de42
+native_color: galera:5 allocation score on galera-1: -INFINITY
60de42
+native_color: galera:5 allocation score on galera-2: -INFINITY
60de42
+native_color: galera:5 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera:5 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera:5 allocation score on messaging-2: -INFINITY
60de42
+native_color: galera:6 allocation score on controller-0: -INFINITY
60de42
+native_color: galera:6 allocation score on controller-1: -INFINITY
60de42
+native_color: galera:6 allocation score on controller-2: -INFINITY
60de42
+native_color: galera:6 allocation score on galera-0: -INFINITY
60de42
+native_color: galera:6 allocation score on galera-1: -INFINITY
60de42
+native_color: galera:6 allocation score on galera-2: -INFINITY
60de42
+native_color: galera:6 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera:6 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera:6 allocation score on messaging-2: -INFINITY
60de42
+native_color: galera:7 allocation score on controller-0: -INFINITY
60de42
+native_color: galera:7 allocation score on controller-1: -INFINITY
60de42
+native_color: galera:7 allocation score on controller-2: -INFINITY
60de42
+native_color: galera:7 allocation score on galera-0: -INFINITY
60de42
+native_color: galera:7 allocation score on galera-1: -INFINITY
60de42
+native_color: galera:7 allocation score on galera-2: -INFINITY
60de42
+native_color: galera:7 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera:7 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera:7 allocation score on messaging-2: -INFINITY
60de42
+native_color: galera:8 allocation score on controller-0: -INFINITY
60de42
+native_color: galera:8 allocation score on controller-1: -INFINITY
60de42
+native_color: galera:8 allocation score on controller-2: -INFINITY
60de42
+native_color: galera:8 allocation score on galera-0: -INFINITY
60de42
+native_color: galera:8 allocation score on galera-1: -INFINITY
60de42
+native_color: galera:8 allocation score on galera-2: -INFINITY
60de42
+native_color: galera:8 allocation score on messaging-0: -INFINITY
60de42
+native_color: galera:8 allocation score on messaging-1: -INFINITY
60de42
+native_color: galera:8 allocation score on messaging-2: -INFINITY
60de42
+native_color: haproxy:0 allocation score on controller-0: -INFINITY
60de42
+native_color: haproxy:0 allocation score on controller-1: -INFINITY
60de42
+native_color: haproxy:0 allocation score on controller-2: -INFINITY
60de42
+native_color: haproxy:0 allocation score on galera-0: -INFINITY
60de42
+native_color: haproxy:0 allocation score on galera-1: -INFINITY
60de42
+native_color: haproxy:0 allocation score on galera-2: -INFINITY
60de42
+native_color: haproxy:0 allocation score on messaging-0: -INFINITY
60de42
+native_color: haproxy:0 allocation score on messaging-1: -INFINITY
60de42
+native_color: haproxy:0 allocation score on messaging-2: -INFINITY
60de42
+native_color: haproxy:1 allocation score on controller-0: INFINITY
60de42
+native_color: haproxy:1 allocation score on controller-1: -INFINITY
60de42
+native_color: haproxy:1 allocation score on controller-2: 0
60de42
+native_color: haproxy:1 allocation score on galera-0: -INFINITY
60de42
+native_color: haproxy:1 allocation score on galera-1: -INFINITY
60de42
+native_color: haproxy:1 allocation score on galera-2: -INFINITY
60de42
+native_color: haproxy:1 allocation score on messaging-0: -INFINITY
60de42
+native_color: haproxy:1 allocation score on messaging-1: -INFINITY
60de42
+native_color: haproxy:1 allocation score on messaging-2: -INFINITY
60de42
+native_color: haproxy:2 allocation score on controller-0: -INFINITY
60de42
+native_color: haproxy:2 allocation score on controller-1: -INFINITY
60de42
+native_color: haproxy:2 allocation score on controller-2: INFINITY
60de42
+native_color: haproxy:2 allocation score on galera-0: -INFINITY
60de42
+native_color: haproxy:2 allocation score on galera-1: -INFINITY
60de42
+native_color: haproxy:2 allocation score on galera-2: -INFINITY
60de42
+native_color: haproxy:2 allocation score on messaging-0: -INFINITY
60de42
+native_color: haproxy:2 allocation score on messaging-1: -INFINITY
60de42
+native_color: haproxy:2 allocation score on messaging-2: -INFINITY
60de42
+native_color: haproxy:3 allocation score on controller-0: -INFINITY
60de42
+native_color: haproxy:3 allocation score on controller-1: -INFINITY
60de42
+native_color: haproxy:3 allocation score on controller-2: -INFINITY
60de42
+native_color: haproxy:3 allocation score on galera-0: -INFINITY
60de42
+native_color: haproxy:3 allocation score on galera-1: -INFINITY
60de42
+native_color: haproxy:3 allocation score on galera-2: -INFINITY
60de42
+native_color: haproxy:3 allocation score on messaging-0: -INFINITY
60de42
+native_color: haproxy:3 allocation score on messaging-1: -INFINITY
60de42
+native_color: haproxy:3 allocation score on messaging-2: -INFINITY
60de42
+native_color: haproxy:4 allocation score on controller-0: -INFINITY
60de42
+native_color: haproxy:4 allocation score on controller-1: -INFINITY
60de42
+native_color: haproxy:4 allocation score on controller-2: -INFINITY
60de42
+native_color: haproxy:4 allocation score on galera-0: -INFINITY
60de42
+native_color: haproxy:4 allocation score on galera-1: -INFINITY
60de42
+native_color: haproxy:4 allocation score on galera-2: -INFINITY
60de42
+native_color: haproxy:4 allocation score on messaging-0: -INFINITY
60de42
+native_color: haproxy:4 allocation score on messaging-1: -INFINITY
60de42
+native_color: haproxy:4 allocation score on messaging-2: -INFINITY
60de42
+native_color: haproxy:5 allocation score on controller-0: -INFINITY
60de42
+native_color: haproxy:5 allocation score on controller-1: -INFINITY
60de42
+native_color: haproxy:5 allocation score on controller-2: -INFINITY
60de42
+native_color: haproxy:5 allocation score on galera-0: -INFINITY
60de42
+native_color: haproxy:5 allocation score on galera-1: -INFINITY
60de42
+native_color: haproxy:5 allocation score on galera-2: -INFINITY
60de42
+native_color: haproxy:5 allocation score on messaging-0: -INFINITY
60de42
+native_color: haproxy:5 allocation score on messaging-1: -INFINITY
60de42
+native_color: haproxy:5 allocation score on messaging-2: -INFINITY
60de42
+native_color: haproxy:6 allocation score on controller-0: -INFINITY
60de42
+native_color: haproxy:6 allocation score on controller-1: -INFINITY
60de42
+native_color: haproxy:6 allocation score on controller-2: -INFINITY
60de42
+native_color: haproxy:6 allocation score on galera-0: -INFINITY
60de42
+native_color: haproxy:6 allocation score on galera-1: -INFINITY
60de42
+native_color: haproxy:6 allocation score on galera-2: -INFINITY
60de42
+native_color: haproxy:6 allocation score on messaging-0: -INFINITY
60de42
+native_color: haproxy:6 allocation score on messaging-1: -INFINITY
60de42
+native_color: haproxy:6 allocation score on messaging-2: -INFINITY
60de42
+native_color: haproxy:7 allocation score on controller-0: -INFINITY
60de42
+native_color: haproxy:7 allocation score on controller-1: -INFINITY
60de42
+native_color: haproxy:7 allocation score on controller-2: -INFINITY
60de42
+native_color: haproxy:7 allocation score on galera-0: -INFINITY
60de42
+native_color: haproxy:7 allocation score on galera-1: -INFINITY
60de42
+native_color: haproxy:7 allocation score on galera-2: -INFINITY
60de42
+native_color: haproxy:7 allocation score on messaging-0: -INFINITY
60de42
+native_color: haproxy:7 allocation score on messaging-1: -INFINITY
60de42
+native_color: haproxy:7 allocation score on messaging-2: -INFINITY
60de42
+native_color: haproxy:8 allocation score on controller-0: -INFINITY
60de42
+native_color: haproxy:8 allocation score on controller-1: -INFINITY
60de42
+native_color: haproxy:8 allocation score on controller-2: -INFINITY
60de42
+native_color: haproxy:8 allocation score on galera-0: -INFINITY
60de42
+native_color: haproxy:8 allocation score on galera-1: -INFINITY
60de42
+native_color: haproxy:8 allocation score on galera-2: -INFINITY
60de42
+native_color: haproxy:8 allocation score on messaging-0: -INFINITY
60de42
+native_color: haproxy:8 allocation score on messaging-1: -INFINITY
60de42
+native_color: haproxy:8 allocation score on messaging-2: -INFINITY
60de42
+native_color: ip-10.0.0.102 allocation score on controller-0: INFINITY
60de42
+native_color: ip-10.0.0.102 allocation score on controller-1: -INFINITY
60de42
+native_color: ip-10.0.0.102 allocation score on controller-2: 0
60de42
+native_color: ip-10.0.0.102 allocation score on galera-0: -INFINITY
60de42
+native_color: ip-10.0.0.102 allocation score on galera-1: -INFINITY
60de42
+native_color: ip-10.0.0.102 allocation score on galera-2: -INFINITY
60de42
+native_color: ip-10.0.0.102 allocation score on messaging-0: -INFINITY
60de42
+native_color: ip-10.0.0.102 allocation score on messaging-1: -INFINITY
60de42
+native_color: ip-10.0.0.102 allocation score on messaging-2: -INFINITY
60de42
+native_color: ip-172.17.1.14 allocation score on controller-0: 0
60de42
+native_color: ip-172.17.1.14 allocation score on controller-1: -INFINITY
60de42
+native_color: ip-172.17.1.14 allocation score on controller-2: 0
60de42
+native_color: ip-172.17.1.14 allocation score on galera-0: -INFINITY
60de42
+native_color: ip-172.17.1.14 allocation score on galera-1: -INFINITY
60de42
+native_color: ip-172.17.1.14 allocation score on galera-2: -INFINITY
60de42
+native_color: ip-172.17.1.14 allocation score on messaging-0: -INFINITY
60de42
+native_color: ip-172.17.1.14 allocation score on messaging-1: -INFINITY
60de42
+native_color: ip-172.17.1.14 allocation score on messaging-2: -INFINITY
60de42
+native_color: ip-172.17.1.17 allocation score on controller-0: 0
60de42
+native_color: ip-172.17.1.17 allocation score on controller-1: -INFINITY
60de42
+native_color: ip-172.17.1.17 allocation score on controller-2: 0
60de42
+native_color: ip-172.17.1.17 allocation score on galera-0: -INFINITY
60de42
+native_color: ip-172.17.1.17 allocation score on galera-1: -INFINITY
60de42
+native_color: ip-172.17.1.17 allocation score on galera-2: -INFINITY
60de42
+native_color: ip-172.17.1.17 allocation score on messaging-0: -INFINITY
60de42
+native_color: ip-172.17.1.17 allocation score on messaging-1: -INFINITY
60de42
+native_color: ip-172.17.1.17 allocation score on messaging-2: -INFINITY
60de42
+native_color: ip-172.17.3.15 allocation score on controller-0: INFINITY
60de42
+native_color: ip-172.17.3.15 allocation score on controller-1: -INFINITY
60de42
+native_color: ip-172.17.3.15 allocation score on controller-2: 0
60de42
+native_color: ip-172.17.3.15 allocation score on galera-0: -INFINITY
60de42
+native_color: ip-172.17.3.15 allocation score on galera-1: -INFINITY
60de42
+native_color: ip-172.17.3.15 allocation score on galera-2: -INFINITY
60de42
+native_color: ip-172.17.3.15 allocation score on messaging-0: -INFINITY
60de42
+native_color: ip-172.17.3.15 allocation score on messaging-1: -INFINITY
60de42
+native_color: ip-172.17.3.15 allocation score on messaging-2: -INFINITY
60de42
+native_color: ip-172.17.4.11 allocation score on controller-0: 0
60de42
+native_color: ip-172.17.4.11 allocation score on controller-1: -INFINITY
60de42
+native_color: ip-172.17.4.11 allocation score on controller-2: 0
60de42
+native_color: ip-172.17.4.11 allocation score on galera-0: -INFINITY
60de42
+native_color: ip-172.17.4.11 allocation score on galera-1: -INFINITY
60de42
+native_color: ip-172.17.4.11 allocation score on galera-2: -INFINITY
60de42
+native_color: ip-172.17.4.11 allocation score on messaging-0: -INFINITY
60de42
+native_color: ip-172.17.4.11 allocation score on messaging-1: -INFINITY
60de42
+native_color: ip-172.17.4.11 allocation score on messaging-2: -INFINITY
60de42
+native_color: ip-192.168.24.6 allocation score on controller-0: INFINITY
60de42
+native_color: ip-192.168.24.6 allocation score on controller-1: -INFINITY
60de42
+native_color: ip-192.168.24.6 allocation score on controller-2: 0
60de42
+native_color: ip-192.168.24.6 allocation score on galera-0: -INFINITY
60de42
+native_color: ip-192.168.24.6 allocation score on galera-1: -INFINITY
60de42
+native_color: ip-192.168.24.6 allocation score on galera-2: -INFINITY
60de42
+native_color: ip-192.168.24.6 allocation score on messaging-0: -INFINITY
60de42
+native_color: ip-192.168.24.6 allocation score on messaging-1: -INFINITY
60de42
+native_color: ip-192.168.24.6 allocation score on messaging-2: -INFINITY
60de42
+native_color: messaging-0 allocation score on controller-0: INFINITY
60de42
+native_color: messaging-0 allocation score on controller-1: 0
60de42
+native_color: messaging-0 allocation score on controller-2: 0
60de42
+native_color: messaging-0 allocation score on galera-0: -INFINITY
60de42
+native_color: messaging-0 allocation score on galera-1: -INFINITY
60de42
+native_color: messaging-0 allocation score on galera-2: -INFINITY
60de42
+native_color: messaging-0 allocation score on messaging-0: -INFINITY
60de42
+native_color: messaging-0 allocation score on messaging-1: -INFINITY
60de42
+native_color: messaging-0 allocation score on messaging-2: -INFINITY
60de42
+native_color: messaging-1 allocation score on controller-0: 0
60de42
+native_color: messaging-1 allocation score on controller-1: INFINITY
60de42
+native_color: messaging-1 allocation score on controller-2: 0
60de42
+native_color: messaging-1 allocation score on galera-0: -INFINITY
60de42
+native_color: messaging-1 allocation score on galera-1: -INFINITY
60de42
+native_color: messaging-1 allocation score on galera-2: -INFINITY
60de42
+native_color: messaging-1 allocation score on messaging-0: -INFINITY
60de42
+native_color: messaging-1 allocation score on messaging-1: -INFINITY
60de42
+native_color: messaging-1 allocation score on messaging-2: -INFINITY
60de42
+native_color: messaging-2 allocation score on controller-0: INFINITY
60de42
+native_color: messaging-2 allocation score on controller-1: 0
60de42
+native_color: messaging-2 allocation score on controller-2: 0
60de42
+native_color: messaging-2 allocation score on galera-0: -INFINITY
60de42
+native_color: messaging-2 allocation score on galera-1: -INFINITY
60de42
+native_color: messaging-2 allocation score on galera-2: -INFINITY
60de42
+native_color: messaging-2 allocation score on messaging-0: -INFINITY
60de42
+native_color: messaging-2 allocation score on messaging-1: -INFINITY
60de42
+native_color: messaging-2 allocation score on messaging-2: -INFINITY
60de42
+native_color: openstack-cinder-volume allocation score on controller-0: INFINITY
60de42
+native_color: openstack-cinder-volume allocation score on controller-1: 0
60de42
+native_color: openstack-cinder-volume allocation score on controller-2: 0
60de42
+native_color: openstack-cinder-volume allocation score on galera-0: -INFINITY
60de42
+native_color: openstack-cinder-volume allocation score on galera-1: -INFINITY
60de42
+native_color: openstack-cinder-volume allocation score on galera-2: -INFINITY
60de42
+native_color: openstack-cinder-volume allocation score on messaging-0: -INFINITY
60de42
+native_color: openstack-cinder-volume allocation score on messaging-1: -INFINITY
60de42
+native_color: openstack-cinder-volume allocation score on messaging-2: -INFINITY
60de42
+native_color: rabbitmq:0 allocation score on controller-0: -INFINITY
60de42
+native_color: rabbitmq:0 allocation score on controller-1: -INFINITY
60de42
+native_color: rabbitmq:0 allocation score on controller-2: -INFINITY
60de42
+native_color: rabbitmq:0 allocation score on galera-0: -INFINITY
60de42
+native_color: rabbitmq:0 allocation score on galera-1: -INFINITY
60de42
+native_color: rabbitmq:0 allocation score on galera-2: -INFINITY
60de42
+native_color: rabbitmq:0 allocation score on messaging-0: 0
60de42
+native_color: rabbitmq:0 allocation score on messaging-1: 0
60de42
+native_color: rabbitmq:0 allocation score on messaging-2: INFINITY
60de42
+native_color: rabbitmq:1 allocation score on controller-0: -INFINITY
60de42
+native_color: rabbitmq:1 allocation score on controller-1: -INFINITY
60de42
+native_color: rabbitmq:1 allocation score on controller-2: -INFINITY
60de42
+native_color: rabbitmq:1 allocation score on galera-0: -INFINITY
60de42
+native_color: rabbitmq:1 allocation score on galera-1: -INFINITY
60de42
+native_color: rabbitmq:1 allocation score on galera-2: -INFINITY
60de42
+native_color: rabbitmq:1 allocation score on messaging-0: INFINITY
60de42
+native_color: rabbitmq:1 allocation score on messaging-1: 0
60de42
+native_color: rabbitmq:1 allocation score on messaging-2: -INFINITY
60de42
+native_color: rabbitmq:2 allocation score on controller-0: -INFINITY
60de42
+native_color: rabbitmq:2 allocation score on controller-1: -INFINITY
60de42
+native_color: rabbitmq:2 allocation score on controller-2: -INFINITY
60de42
+native_color: rabbitmq:2 allocation score on galera-0: -INFINITY
60de42
+native_color: rabbitmq:2 allocation score on galera-1: -INFINITY
60de42
+native_color: rabbitmq:2 allocation score on galera-2: -INFINITY
60de42
+native_color: rabbitmq:2 allocation score on messaging-0: -INFINITY
60de42
+native_color: rabbitmq:2 allocation score on messaging-1: INFINITY
60de42
+native_color: rabbitmq:2 allocation score on messaging-2: -INFINITY
60de42
+native_color: rabbitmq:3 allocation score on controller-0: -INFINITY
60de42
+native_color: rabbitmq:3 allocation score on controller-1: -INFINITY
60de42
+native_color: rabbitmq:3 allocation score on controller-2: -INFINITY
60de42
+native_color: rabbitmq:3 allocation score on galera-0: -INFINITY
60de42
+native_color: rabbitmq:3 allocation score on galera-1: -INFINITY
60de42
+native_color: rabbitmq:3 allocation score on galera-2: -INFINITY
60de42
+native_color: rabbitmq:3 allocation score on messaging-0: -INFINITY
60de42
+native_color: rabbitmq:3 allocation score on messaging-1: -INFINITY
60de42
+native_color: rabbitmq:3 allocation score on messaging-2: -INFINITY
60de42
+native_color: rabbitmq:4 allocation score on controller-0: -INFINITY
60de42
+native_color: rabbitmq:4 allocation score on controller-1: -INFINITY
60de42
+native_color: rabbitmq:4 allocation score on controller-2: -INFINITY
60de42
+native_color: rabbitmq:4 allocation score on galera-0: -INFINITY
60de42
+native_color: rabbitmq:4 allocation score on galera-1: -INFINITY
60de42
+native_color: rabbitmq:4 allocation score on galera-2: -INFINITY
60de42
+native_color: rabbitmq:4 allocation score on messaging-0: -INFINITY
60de42
+native_color: rabbitmq:4 allocation score on messaging-1: -INFINITY
60de42
+native_color: rabbitmq:4 allocation score on messaging-2: -INFINITY
60de42
+native_color: rabbitmq:5 allocation score on controller-0: -INFINITY
60de42
+native_color: rabbitmq:5 allocation score on controller-1: -INFINITY
60de42
+native_color: rabbitmq:5 allocation score on controller-2: -INFINITY
60de42
+native_color: rabbitmq:5 allocation score on galera-0: -INFINITY
60de42
+native_color: rabbitmq:5 allocation score on galera-1: -INFINITY
60de42
+native_color: rabbitmq:5 allocation score on galera-2: -INFINITY
60de42
+native_color: rabbitmq:5 allocation score on messaging-0: -INFINITY
60de42
+native_color: rabbitmq:5 allocation score on messaging-1: -INFINITY
60de42
+native_color: rabbitmq:5 allocation score on messaging-2: -INFINITY
60de42
+native_color: rabbitmq:6 allocation score on controller-0: -INFINITY
60de42
+native_color: rabbitmq:6 allocation score on controller-1: -INFINITY
60de42
+native_color: rabbitmq:6 allocation score on controller-2: -INFINITY
60de42
+native_color: rabbitmq:6 allocation score on galera-0: -INFINITY
60de42
+native_color: rabbitmq:6 allocation score on galera-1: -INFINITY
60de42
+native_color: rabbitmq:6 allocation score on galera-2: -INFINITY
60de42
+native_color: rabbitmq:6 allocation score on messaging-0: -INFINITY
60de42
+native_color: rabbitmq:6 allocation score on messaging-1: -INFINITY
60de42
+native_color: rabbitmq:6 allocation score on messaging-2: -INFINITY
60de42
+native_color: rabbitmq:7 allocation score on controller-0: -INFINITY
60de42
+native_color: rabbitmq:7 allocation score on controller-1: -INFINITY
60de42
+native_color: rabbitmq:7 allocation score on controller-2: -INFINITY
60de42
+native_color: rabbitmq:7 allocation score on galera-0: -INFINITY
60de42
+native_color: rabbitmq:7 allocation score on galera-1: -INFINITY
60de42
+native_color: rabbitmq:7 allocation score on galera-2: -INFINITY
60de42
+native_color: rabbitmq:7 allocation score on messaging-0: -INFINITY
60de42
+native_color: rabbitmq:7 allocation score on messaging-1: -INFINITY
60de42
+native_color: rabbitmq:7 allocation score on messaging-2: -INFINITY
60de42
+native_color: rabbitmq:8 allocation score on controller-0: -INFINITY
60de42
+native_color: rabbitmq:8 allocation score on controller-1: -INFINITY
60de42
+native_color: rabbitmq:8 allocation score on controller-2: -INFINITY
60de42
+native_color: rabbitmq:8 allocation score on galera-0: -INFINITY
60de42
+native_color: rabbitmq:8 allocation score on galera-1: -INFINITY
60de42
+native_color: rabbitmq:8 allocation score on galera-2: -INFINITY
60de42
+native_color: rabbitmq:8 allocation score on messaging-0: -INFINITY
60de42
+native_color: rabbitmq:8 allocation score on messaging-1: -INFINITY
60de42
+native_color: rabbitmq:8 allocation score on messaging-2: -INFINITY
60de42
+native_color: redis:0 allocation score on controller-0: -INFINITY
60de42
+native_color: redis:0 allocation score on controller-1: -INFINITY
60de42
+native_color: redis:0 allocation score on controller-2: -INFINITY
60de42
+native_color: redis:0 allocation score on galera-0: -INFINITY
60de42
+native_color: redis:0 allocation score on galera-1: -INFINITY
60de42
+native_color: redis:0 allocation score on galera-2: -INFINITY
60de42
+native_color: redis:0 allocation score on messaging-0: -INFINITY
60de42
+native_color: redis:0 allocation score on messaging-1: -INFINITY
60de42
+native_color: redis:0 allocation score on messaging-2: -INFINITY
60de42
+native_color: redis:1 allocation score on controller-0: INFINITY
60de42
+native_color: redis:1 allocation score on controller-1: -INFINITY
60de42
+native_color: redis:1 allocation score on controller-2: 0
60de42
+native_color: redis:1 allocation score on galera-0: -INFINITY
60de42
+native_color: redis:1 allocation score on galera-1: -INFINITY
60de42
+native_color: redis:1 allocation score on galera-2: -INFINITY
60de42
+native_color: redis:1 allocation score on messaging-0: -INFINITY
60de42
+native_color: redis:1 allocation score on messaging-1: -INFINITY
60de42
+native_color: redis:1 allocation score on messaging-2: -INFINITY
60de42
+native_color: redis:2 allocation score on controller-0: -INFINITY
60de42
+native_color: redis:2 allocation score on controller-1: -INFINITY
60de42
+native_color: redis:2 allocation score on controller-2: INFINITY
60de42
+native_color: redis:2 allocation score on galera-0: -INFINITY
60de42
+native_color: redis:2 allocation score on galera-1: -INFINITY
60de42
+native_color: redis:2 allocation score on galera-2: -INFINITY
60de42
+native_color: redis:2 allocation score on messaging-0: -INFINITY
60de42
+native_color: redis:2 allocation score on messaging-1: -INFINITY
60de42
+native_color: redis:2 allocation score on messaging-2: -INFINITY
60de42
+native_color: redis:3 allocation score on controller-0: -INFINITY
60de42
+native_color: redis:3 allocation score on controller-1: -INFINITY
60de42
+native_color: redis:3 allocation score on controller-2: -INFINITY
60de42
+native_color: redis:3 allocation score on galera-0: -INFINITY
60de42
+native_color: redis:3 allocation score on galera-1: -INFINITY
60de42
+native_color: redis:3 allocation score on galera-2: -INFINITY
60de42
+native_color: redis:3 allocation score on messaging-0: -INFINITY
60de42
+native_color: redis:3 allocation score on messaging-1: -INFINITY
60de42
+native_color: redis:3 allocation score on messaging-2: -INFINITY
60de42
+native_color: redis:4 allocation score on controller-0: -INFINITY
60de42
+native_color: redis:4 allocation score on controller-1: -INFINITY
60de42
+native_color: redis:4 allocation score on controller-2: -INFINITY
60de42
+native_color: redis:4 allocation score on galera-0: -INFINITY
60de42
+native_color: redis:4 allocation score on galera-1: -INFINITY
60de42
+native_color: redis:4 allocation score on galera-2: -INFINITY
60de42
+native_color: redis:4 allocation score on messaging-0: -INFINITY
60de42
+native_color: redis:4 allocation score on messaging-1: -INFINITY
60de42
+native_color: redis:4 allocation score on messaging-2: -INFINITY
60de42
+native_color: redis:5 allocation score on controller-0: -INFINITY
60de42
+native_color: redis:5 allocation score on controller-1: -INFINITY
60de42
+native_color: redis:5 allocation score on controller-2: -INFINITY
60de42
+native_color: redis:5 allocation score on galera-0: -INFINITY
60de42
+native_color: redis:5 allocation score on galera-1: -INFINITY
60de42
+native_color: redis:5 allocation score on galera-2: -INFINITY
60de42
+native_color: redis:5 allocation score on messaging-0: -INFINITY
60de42
+native_color: redis:5 allocation score on messaging-1: -INFINITY
60de42
+native_color: redis:5 allocation score on messaging-2: -INFINITY
60de42
+native_color: redis:6 allocation score on controller-0: -INFINITY
60de42
+native_color: redis:6 allocation score on controller-1: -INFINITY
60de42
+native_color: redis:6 allocation score on controller-2: -INFINITY
60de42
+native_color: redis:6 allocation score on galera-0: -INFINITY
60de42
+native_color: redis:6 allocation score on galera-1: -INFINITY
60de42
+native_color: redis:6 allocation score on galera-2: -INFINITY
60de42
+native_color: redis:6 allocation score on messaging-0: -INFINITY
60de42
+native_color: redis:6 allocation score on messaging-1: -INFINITY
60de42
+native_color: redis:6 allocation score on messaging-2: -INFINITY
60de42
+native_color: redis:7 allocation score on controller-0: -INFINITY
60de42
+native_color: redis:7 allocation score on controller-1: -INFINITY
60de42
+native_color: redis:7 allocation score on controller-2: -INFINITY
60de42
+native_color: redis:7 allocation score on galera-0: -INFINITY
60de42
+native_color: redis:7 allocation score on galera-1: -INFINITY
60de42
+native_color: redis:7 allocation score on galera-2: -INFINITY
60de42
+native_color: redis:7 allocation score on messaging-0: -INFINITY
60de42
+native_color: redis:7 allocation score on messaging-1: -INFINITY
60de42
+native_color: redis:7 allocation score on messaging-2: -INFINITY
60de42
+native_color: redis:8 allocation score on controller-0: -INFINITY
60de42
+native_color: redis:8 allocation score on controller-1: -INFINITY
60de42
+native_color: redis:8 allocation score on controller-2: -INFINITY
60de42
+native_color: redis:8 allocation score on galera-0: -INFINITY
60de42
+native_color: redis:8 allocation score on galera-1: -INFINITY
60de42
+native_color: redis:8 allocation score on galera-2: -INFINITY
60de42
+native_color: redis:8 allocation score on messaging-0: -INFINITY
60de42
+native_color: redis:8 allocation score on messaging-1: -INFINITY
60de42
+native_color: redis:8 allocation score on messaging-2: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-5254005bdbb5 allocation score on controller-0: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-5254005bdbb5 allocation score on controller-1: INFINITY
60de42
+native_color: stonith-fence_ipmilan-5254005bdbb5 allocation score on controller-2: 0
60de42
+native_color: stonith-fence_ipmilan-5254005bdbb5 allocation score on galera-0: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-5254005bdbb5 allocation score on galera-1: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-5254005bdbb5 allocation score on galera-2: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-5254005bdbb5 allocation score on messaging-0: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-5254005bdbb5 allocation score on messaging-1: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-5254005bdbb5 allocation score on messaging-2: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400b4f6bd allocation score on controller-0: INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400b4f6bd allocation score on controller-1: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400b4f6bd allocation score on controller-2: 0
60de42
+native_color: stonith-fence_ipmilan-525400b4f6bd allocation score on galera-0: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400b4f6bd allocation score on galera-1: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400b4f6bd allocation score on galera-2: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400b4f6bd allocation score on messaging-0: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400b4f6bd allocation score on messaging-1: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400b4f6bd allocation score on messaging-2: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400bbf613 allocation score on controller-0: INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400bbf613 allocation score on controller-1: 0
60de42
+native_color: stonith-fence_ipmilan-525400bbf613 allocation score on controller-2: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400bbf613 allocation score on galera-0: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400bbf613 allocation score on galera-1: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400bbf613 allocation score on galera-2: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400bbf613 allocation score on messaging-0: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400bbf613 allocation score on messaging-1: -INFINITY
60de42
+native_color: stonith-fence_ipmilan-525400bbf613 allocation score on messaging-2: -INFINITY
60de42
+redis:0 promotion score on none: 0
60de42
+redis:1 promotion score on controller-0: 1
60de42
+redis:2 promotion score on controller-2: 1
60de42
+redis:3 promotion score on none: 0
60de42
+redis:4 promotion score on none: 0
60de42
+redis:5 promotion score on none: 0
60de42
+redis:6 promotion score on none: 0
60de42
+redis:7 promotion score on none: 0
60de42
+redis:8 promotion score on none: 0
60de42
diff --git a/pengine/test10/remote-recovery.summary b/pengine/test10/remote-recovery.summary
60de42
new file mode 100644
60de42
index 0000000..251febc
60de42
--- /dev/null
60de42
+++ b/pengine/test10/remote-recovery.summary
60de42
@@ -0,0 +1,164 @@
60de42
+Using the original execution date of: 2017-05-03 13:33:24Z
60de42
+
60de42
+Current cluster status:
60de42
+Node controller-1 (2): UNCLEAN (offline)
60de42
+Online: [ controller-0 controller-2 ]
60de42
+RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
60de42
+
60de42
+ messaging-0	(ocf::pacemaker:remote):	Started controller-0
60de42
+ messaging-1	(ocf::pacemaker:remote):	Started controller-1 (UNCLEAN)
60de42
+ messaging-2	(ocf::pacemaker:remote):	Started controller-0
60de42
+ galera-0	(ocf::pacemaker:remote):	Started controller-1 (UNCLEAN)
60de42
+ galera-1	(ocf::pacemaker:remote):	Started controller-0
60de42
+ galera-2	(ocf::pacemaker:remote):	Started controller-1 (UNCLEAN)
60de42
+ Clone Set: rabbitmq-clone [rabbitmq]
60de42
+     Started: [ messaging-0 messaging-1 messaging-2 ]
60de42
+     Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ]
60de42
+ Master/Slave Set: galera-master [galera]
60de42
+     Masters: [ galera-0 galera-1 galera-2 ]
60de42
+     Stopped: [ controller-0 controller-1 controller-2 messaging-0 messaging-1 messaging-2 ]
60de42
+ Master/Slave Set: redis-master [redis]
60de42
+     redis	(ocf::heartbeat:redis):	Slave controller-1 (UNCLEAN)
60de42
+     Masters: [ controller-0 ]
60de42
+     Slaves: [ controller-2 ]
60de42
+     Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
60de42
+ ip-192.168.24.6	(ocf::heartbeat:IPaddr2):	Started controller-0
60de42
+ ip-10.0.0.102	(ocf::heartbeat:IPaddr2):	Started controller-0
60de42
+ ip-172.17.1.14	(ocf::heartbeat:IPaddr2):	Started controller-1 (UNCLEAN)
60de42
+ ip-172.17.1.17	(ocf::heartbeat:IPaddr2):	Started controller-1 (UNCLEAN)
60de42
+ ip-172.17.3.15	(ocf::heartbeat:IPaddr2):	Started controller-0
60de42
+ ip-172.17.4.11	(ocf::heartbeat:IPaddr2):	Started controller-1 (UNCLEAN)
60de42
+ Clone Set: haproxy-clone [haproxy]
60de42
+     haproxy	(systemd:haproxy):	Started controller-1 (UNCLEAN)
60de42
+     Started: [ controller-0 controller-2 ]
60de42
+     Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
60de42
+ openstack-cinder-volume	(systemd:openstack-cinder-volume):	Started controller-0
60de42
+ stonith-fence_ipmilan-525400bbf613	(stonith:fence_ipmilan):	Started controller-0
60de42
+ stonith-fence_ipmilan-525400b4f6bd	(stonith:fence_ipmilan):	Started controller-0
60de42
+ stonith-fence_ipmilan-5254005bdbb5	(stonith:fence_ipmilan):	Started controller-1 (UNCLEAN)
60de42
+
60de42
+Transition Summary:
60de42
+ * Move    messaging-1	(Started controller-1 -> controller-2)
60de42
+ * Move    galera-0	(Started controller-1 -> controller-2)
60de42
+ * Move    galera-2	(Started controller-1 -> controller-2)
60de42
+ * Restart rabbitmq:2	(Started messaging-1)
60de42
+ * Restart galera:1	(Master galera-2)
60de42
+ * Restart galera:2	(Master galera-0)
60de42
+ * Stop    redis:0	(controller-1)
60de42
+ * Move    ip-172.17.1.14	(Started controller-1 -> controller-2)
60de42
+ * Move    ip-172.17.1.17	(Started controller-1 -> controller-2)
60de42
+ * Move    ip-172.17.4.11	(Started controller-1 -> controller-2)
60de42
+ * Stop    haproxy:0	(controller-1)
60de42
+ * Restart stonith-fence_ipmilan-525400bbf613	(Started controller-0)
60de42
+ * Restart stonith-fence_ipmilan-525400b4f6bd	(Started controller-0)
60de42
+ * Move    stonith-fence_ipmilan-5254005bdbb5	(Started controller-1 -> controller-2)
60de42
+
60de42
+Executing cluster transition:
60de42
+ * Pseudo action:   rabbitmq-clone_stop_0
60de42
+ * Pseudo action:   galera-master_demote_0
60de42
+ * Pseudo action:   redis-master_pre_notify_stop_0
60de42
+ * Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0
60de42
+ * Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0
60de42
+ * Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0
60de42
+ * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0
60de42
+ * Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0
60de42
+ * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0
60de42
+ * Pseudo action:   stonith-fence_ipmilan-5254005bdbb5_stop_0
60de42
+ * Fencing controller-1 (reboot)
60de42
+ * Pseudo action:   stonith_complete
60de42
+ * Pseudo action:   messaging-1_stop_0
60de42
+ * Pseudo action:   galera-0_stop_0
60de42
+ * Pseudo action:   galera-2_stop_0
60de42
+ * Pseudo action:   redis_post_notify_stop_0
60de42
+ * Resource action: redis           notify on controller-0
60de42
+ * Resource action: redis           notify on controller-2
60de42
+ * Pseudo action:   redis-master_confirmed-pre_notify_stop_0
60de42
+ * Pseudo action:   redis-master_stop_0
60de42
+ * Pseudo action:   haproxy-clone_stop_0
60de42
+ * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2
60de42
+ * Resource action: messaging-1     start on controller-2
60de42
+ * Resource action: galera-0        start on controller-2
60de42
+ * Resource action: galera-2        start on controller-2
60de42
+ * Resource action: rabbitmq        stop on messaging-1
60de42
+ * Pseudo action:   rabbitmq-clone_stopped_0
60de42
+ * Pseudo action:   rabbitmq-clone_start_0
60de42
+ * Resource action: galera          demote on galera-0
60de42
+ * Pseudo action:   redis_stop_0
60de42
+ * Pseudo action:   redis-master_stopped_0
60de42
+ * Pseudo action:   haproxy_stop_0
60de42
+ * Pseudo action:   haproxy-clone_stopped_0
60de42
+ * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2
60de42
+ * Resource action: messaging-1     monitor=20000 on controller-2
60de42
+ * Resource action: galera-0        monitor=20000 on controller-2
60de42
+ * Resource action: galera-2        monitor=20000 on controller-2
60de42
+ * Resource action: rabbitmq        start on messaging-1
60de42
+ * Resource action: rabbitmq        monitor=10000 on messaging-1
60de42
+ * Pseudo action:   rabbitmq-clone_running_0
60de42
+ * Resource action: galera          demote on galera-2
60de42
+ * Pseudo action:   galera-master_demoted_0
60de42
+ * Pseudo action:   galera-master_stop_0
60de42
+ * Pseudo action:   redis-master_post_notify_stopped_0
60de42
+ * Pseudo action:   ip-172.17.1.14_stop_0
60de42
+ * Pseudo action:   ip-172.17.1.17_stop_0
60de42
+ * Pseudo action:   ip-172.17.4.11_stop_0
60de42
+ * Resource action: galera          stop on galera-0
60de42
+ * Resource action: redis           notify on controller-0
60de42
+ * Resource action: redis           notify on controller-2
60de42
+ * Pseudo action:   redis-master_confirmed-post_notify_stopped_0
60de42
+ * Resource action: ip-172.17.1.14  start on controller-2
60de42
+ * Resource action: ip-172.17.1.17  start on controller-2
60de42
+ * Resource action: ip-172.17.4.11  start on controller-2
60de42
+ * Resource action: galera          stop on galera-2
60de42
+ * Pseudo action:   galera-master_stopped_0
60de42
+ * Pseudo action:   galera-master_start_0
60de42
+ * Pseudo action:   redis_notified_0
60de42
+ * Resource action: ip-172.17.1.14  monitor=10000 on controller-2
60de42
+ * Resource action: ip-172.17.1.17  monitor=10000 on controller-2
60de42
+ * Resource action: ip-172.17.4.11  monitor=10000 on controller-2
60de42
+ * Pseudo action:   all_stopped
60de42
+ * Resource action: galera          start on galera-2
60de42
+ * Resource action: galera          start on galera-0
60de42
+ * Pseudo action:   galera-master_running_0
60de42
+ * Pseudo action:   galera-master_promote_0
60de42
+ * Resource action: galera          promote on galera-2
60de42
+ * Resource action: galera          monitor=10000 on galera-2
60de42
+ * Resource action: galera          promote on galera-0
60de42
+ * Resource action: galera          monitor=10000 on galera-0
60de42
+ * Pseudo action:   galera-master_promoted_0
60de42
+Using the original execution date of: 2017-05-03 13:33:24Z
60de42
+
60de42
+Revised cluster status:
60de42
+Online: [ controller-0 controller-2 ]
60de42
+OFFLINE: [ controller-1 ]
60de42
+RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
60de42
+
60de42
+ messaging-0	(ocf::pacemaker:remote):	Started controller-0
60de42
+ messaging-1	(ocf::pacemaker:remote):	Started controller-2
60de42
+ messaging-2	(ocf::pacemaker:remote):	Started controller-0
60de42
+ galera-0	(ocf::pacemaker:remote):	Started controller-2
60de42
+ galera-1	(ocf::pacemaker:remote):	Started controller-0
60de42
+ galera-2	(ocf::pacemaker:remote):	Started controller-2
60de42
+ Clone Set: rabbitmq-clone [rabbitmq]
60de42
+     Started: [ messaging-0 messaging-1 messaging-2 ]
60de42
+     Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ]
60de42
+ Master/Slave Set: galera-master [galera]
60de42
+     Masters: [ galera-0 galera-1 galera-2 ]
60de42
+     Stopped: [ controller-0 controller-1 controller-2 messaging-0 messaging-1 messaging-2 ]
60de42
+ Master/Slave Set: redis-master [redis]
60de42
+     Masters: [ controller-0 ]
60de42
+     Slaves: [ controller-2 ]
60de42
+     Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
60de42
+ ip-192.168.24.6	(ocf::heartbeat:IPaddr2):	Started controller-0
60de42
+ ip-10.0.0.102	(ocf::heartbeat:IPaddr2):	Started controller-0
60de42
+ ip-172.17.1.14	(ocf::heartbeat:IPaddr2):	Started controller-2
60de42
+ ip-172.17.1.17	(ocf::heartbeat:IPaddr2):	Started controller-2
60de42
+ ip-172.17.3.15	(ocf::heartbeat:IPaddr2):	Started controller-0
60de42
+ ip-172.17.4.11	(ocf::heartbeat:IPaddr2):	Started controller-2
60de42
+ Clone Set: haproxy-clone [haproxy]
60de42
+     Started: [ controller-0 controller-2 ]
60de42
+     Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
60de42
+ openstack-cinder-volume	(systemd:openstack-cinder-volume):	Started controller-0
60de42
+ stonith-fence_ipmilan-525400bbf613	(stonith:fence_ipmilan):	Started controller-0
60de42
+ stonith-fence_ipmilan-525400b4f6bd	(stonith:fence_ipmilan):	Started controller-0
60de42
+ stonith-fence_ipmilan-5254005bdbb5	(stonith:fence_ipmilan):	Started controller-2
60de42
+
60de42
diff --git a/pengine/test10/remote-recovery.xml b/pengine/test10/remote-recovery.xml
60de42
new file mode 100644
60de42
index 0000000..f921dfe
60de42
--- /dev/null
60de42
+++ b/pengine/test10/remote-recovery.xml
60de42
@@ -0,0 +1,739 @@
60de42
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.5" epoch="177" num_updates="291" admin_epoch="0" cib-last-written="Wed May  3 13:24:28 2017" update-origin="controller-0" update-client="crm_attribute" update-user="root" have-quorum="1" dc-uuid="1" execution-date="1493818404">
60de42
+  <configuration>
60de42
+    <crm_config>
60de42
+      <cluster_property_set id="cib-bootstrap-options">
60de42
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
60de42
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.15-11.el7_3.4-e174ec8"/>
60de42
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
60de42
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
60de42
+        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
60de42
+        <nvpair id="cib-bootstrap-options-cluster-recheck-interval" name="cluster-recheck-interval" value="60s"/>
60de42
+        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
60de42
+        <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1493817755"/>
60de42
+      </cluster_property_set>
60de42
+      <cluster_property_set id="redis_replication">
60de42
+        <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="controller-0"/>
60de42
+      </cluster_property_set>
60de42
+    </crm_config>
60de42
+    <nodes>
60de42
+      <node id="1" uname="controller-0">
60de42
+        <instance_attributes id="nodes-1">
60de42
+          <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
60de42
+          <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
60de42
+          <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
60de42
+        </instance_attributes>
60de42
+      </node>
60de42
+      <node id="2" uname="controller-1">
60de42
+        <instance_attributes id="nodes-2">
60de42
+          <nvpair id="nodes-2-cinder-volume-role" name="cinder-volume-role" value="true"/>
60de42
+          <nvpair id="nodes-2-haproxy-role" name="haproxy-role" value="true"/>
60de42
+          <nvpair id="nodes-2-redis-role" name="redis-role" value="true"/>
60de42
+        </instance_attributes>
60de42
+      </node>
60de42
+      <node id="3" uname="controller-2">
60de42
+        <instance_attributes id="nodes-3">
60de42
+          <nvpair id="nodes-3-cinder-volume-role" name="cinder-volume-role" value="true"/>
60de42
+          <nvpair id="nodes-3-haproxy-role" name="haproxy-role" value="true"/>
60de42
+          <nvpair id="nodes-3-redis-role" name="redis-role" value="true"/>
60de42
+        </instance_attributes>
60de42
+      </node>
60de42
+      <node id="messaging-1" type="remote" uname="messaging-1">
60de42
+        <instance_attributes id="nodes-messaging-1">
60de42
+          <nvpair id="nodes-messaging-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
60de42
+          <nvpair id="nodes-messaging-1-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@messaging-1"/>
60de42
+        </instance_attributes>
60de42
+      </node>
60de42
+      <node id="galera-1" type="remote" uname="galera-1">
60de42
+        <instance_attributes id="nodes-galera-1">
60de42
+          <nvpair id="nodes-galera-1-galera-role" name="galera-role" value="true"/>
60de42
+        </instance_attributes>
60de42
+      </node>
60de42
+      <node id="messaging-0" type="remote" uname="messaging-0">
60de42
+        <instance_attributes id="nodes-messaging-0">
60de42
+          <nvpair id="nodes-messaging-0-rabbitmq-role" name="rabbitmq-role" value="true"/>
60de42
+          <nvpair id="nodes-messaging-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@messaging-0"/>
60de42
+        </instance_attributes>
60de42
+      </node>
60de42
+      <node id="galera-2" type="remote" uname="galera-2">
60de42
+        <instance_attributes id="nodes-galera-2">
60de42
+          <nvpair id="nodes-galera-2-galera-role" name="galera-role" value="true"/>
60de42
+        </instance_attributes>
60de42
+      </node>
60de42
+      <node id="messaging-2" type="remote" uname="messaging-2">
60de42
+        <instance_attributes id="nodes-messaging-2">
60de42
+          <nvpair id="nodes-messaging-2-rabbitmq-role" name="rabbitmq-role" value="true"/>
60de42
+          <nvpair id="nodes-messaging-2-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@messaging-2"/>
60de42
+        </instance_attributes>
60de42
+      </node>
60de42
+      <node id="galera-0" type="remote" uname="galera-0">
60de42
+        <instance_attributes id="nodes-galera-0">
60de42
+          <nvpair id="nodes-galera-0-galera-role" name="galera-role" value="true"/>
60de42
+        </instance_attributes>
60de42
+      </node>
60de42
+    </nodes>
60de42
+    <resources>
60de42
+      <primitive class="ocf" id="messaging-0" provider="pacemaker" type="remote">
60de42
+        <instance_attributes id="messaging-0-instance_attributes">
60de42
+          <nvpair id="messaging-0-instance_attributes-reconnect_interval" name="reconnect_interval" value="60"/>
60de42
+        </instance_attributes>
60de42
+        <operations>
60de42
+          <op id="messaging-0-start-interval-0s" interval="0s" name="start" timeout="60"/>
60de42
+          <op id="messaging-0-stop-interval-0s" interval="0s" name="stop" timeout="60"/>
60de42
+          <op id="messaging-0-monitor-interval-20" interval="20" name="monitor"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <primitive class="ocf" id="messaging-1" provider="pacemaker" type="remote">
60de42
+        <instance_attributes id="messaging-1-instance_attributes">
60de42
+          <nvpair id="messaging-1-instance_attributes-reconnect_interval" name="reconnect_interval" value="60"/>
60de42
+        </instance_attributes>
60de42
+        <operations>
60de42
+          <op id="messaging-1-start-interval-0s" interval="0s" name="start" timeout="60"/>
60de42
+          <op id="messaging-1-stop-interval-0s" interval="0s" name="stop" timeout="60"/>
60de42
+          <op id="messaging-1-monitor-interval-20" interval="20" name="monitor"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <primitive class="ocf" id="messaging-2" provider="pacemaker" type="remote">
60de42
+        <instance_attributes id="messaging-2-instance_attributes">
60de42
+          <nvpair id="messaging-2-instance_attributes-reconnect_interval" name="reconnect_interval" value="60"/>
60de42
+        </instance_attributes>
60de42
+        <operations>
60de42
+          <op id="messaging-2-start-interval-0s" interval="0s" name="start" timeout="60"/>
60de42
+          <op id="messaging-2-stop-interval-0s" interval="0s" name="stop" timeout="60"/>
60de42
+          <op id="messaging-2-monitor-interval-20" interval="20" name="monitor"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <primitive class="ocf" id="galera-0" provider="pacemaker" type="remote">
60de42
+        <instance_attributes id="galera-0-instance_attributes">
60de42
+          <nvpair id="galera-0-instance_attributes-reconnect_interval" name="reconnect_interval" value="60"/>
60de42
+        </instance_attributes>
60de42
+        <operations>
60de42
+          <op id="galera-0-start-interval-0s" interval="0s" name="start" timeout="60"/>
60de42
+          <op id="galera-0-stop-interval-0s" interval="0s" name="stop" timeout="60"/>
60de42
+          <op id="galera-0-monitor-interval-20" interval="20" name="monitor"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <primitive class="ocf" id="galera-1" provider="pacemaker" type="remote">
60de42
+        <instance_attributes id="galera-1-instance_attributes">
60de42
+          <nvpair id="galera-1-instance_attributes-reconnect_interval" name="reconnect_interval" value="60"/>
60de42
+        </instance_attributes>
60de42
+        <operations>
60de42
+          <op id="galera-1-start-interval-0s" interval="0s" name="start" timeout="60"/>
60de42
+          <op id="galera-1-stop-interval-0s" interval="0s" name="stop" timeout="60"/>
60de42
+          <op id="galera-1-monitor-interval-20" interval="20" name="monitor"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <primitive class="ocf" id="galera-2" provider="pacemaker" type="remote">
60de42
+        <instance_attributes id="galera-2-instance_attributes">
60de42
+          <nvpair id="galera-2-instance_attributes-reconnect_interval" name="reconnect_interval" value="60"/>
60de42
+        </instance_attributes>
60de42
+        <operations>
60de42
+          <op id="galera-2-start-interval-0s" interval="0s" name="start" timeout="60"/>
60de42
+          <op id="galera-2-stop-interval-0s" interval="0s" name="stop" timeout="60"/>
60de42
+          <op id="galera-2-monitor-interval-20" interval="20" name="monitor"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <clone id="rabbitmq-clone">
60de42
+        <primitive class="ocf" id="rabbitmq" provider="heartbeat" type="rabbitmq-cluster">
60de42
+          <instance_attributes id="rabbitmq-instance_attributes">
60de42
+            <nvpair id="rabbitmq-instance_attributes-set_policy" name="set_policy" value="ha-all ^(?!amq\.).* {"ha-mode":"exactly","ha-params":2}"/>
60de42
+          </instance_attributes>
60de42
+          <meta_attributes id="rabbitmq-meta_attributes">
60de42
+            <nvpair id="rabbitmq-meta_attributes-notify" name="notify" value="true"/>
60de42
+          </meta_attributes>
60de42
+          <operations>
60de42
+            <op id="rabbitmq-monitor-interval-10" interval="10" name="monitor" timeout="40"/>
60de42
+            <op id="rabbitmq-start-interval-0s" interval="0s" name="start" timeout="200s"/>
60de42
+            <op id="rabbitmq-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
60de42
+          </operations>
60de42
+        </primitive>
60de42
+        <meta_attributes id="rabbitmq-clone-meta_attributes">
60de42
+          <nvpair id="rabbitmq-clone-meta_attributes-interleave" name="interleave" value="true"/>
60de42
+          <nvpair id="rabbitmq-clone-meta_attributes-ordered" name="ordered" value="true"/>
60de42
+        </meta_attributes>
60de42
+      </clone>
60de42
+      <master id="galera-master">
60de42
+        <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
60de42
+          <instance_attributes id="galera-instance_attributes">
60de42
+            <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
60de42
+            <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
60de42
+            <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://galera-0,galera-1,galera-2"/>
60de42
+          </instance_attributes>
60de42
+          <operations>
60de42
+            <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
60de42
+            <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
60de42
+            <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
60de42
+            <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
60de42
+            <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
60de42
+            <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
60de42
+            <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
60de42
+          </operations>
60de42
+        </primitive>
60de42
+        <meta_attributes id="galera-master-meta_attributes">
60de42
+          <nvpair id="galera-master-meta_attributes-ordered" name="ordered" value="true"/>
60de42
+          <nvpair id="galera-master-meta_attributes-master-max" name="master-max" value="3"/>
60de42
+        </meta_attributes>
60de42
+      </master>
60de42
+      <master id="redis-master">
60de42
+        <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
60de42
+          <instance_attributes id="redis-instance_attributes">
60de42
+            <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
60de42
+          </instance_attributes>
60de42
+          <operations>
60de42
+            <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
60de42
+            <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
60de42
+            <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
60de42
+            <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
60de42
+            <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
60de42
+            <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
60de42
+            <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
60de42
+          </operations>
60de42
+        </primitive>
60de42
+        <meta_attributes id="redis-master-meta_attributes">
60de42
+          <nvpair id="redis-master-meta_attributes-interleave" name="interleave" value="true"/>
60de42
+          <nvpair id="redis-master-meta_attributes-ordered" name="ordered" value="true"/>
60de42
+          <nvpair id="redis-master-meta_attributes-notify" name="notify" value="true"/>
60de42
+        </meta_attributes>
60de42
+      </master>
60de42
+      <primitive class="ocf" id="ip-192.168.24.6" provider="heartbeat" type="IPaddr2">
60de42
+        <instance_attributes id="ip-192.168.24.6-instance_attributes">
60de42
+          <nvpair id="ip-192.168.24.6-instance_attributes-ip" name="ip" value="192.168.24.6"/>
60de42
+          <nvpair id="ip-192.168.24.6-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
60de42
+        </instance_attributes>
60de42
+        <meta_attributes id="ip-192.168.24.6-meta_attributes"/>
60de42
+        <operations>
60de42
+          <op id="ip-192.168.24.6-start-interval-0s" interval="0s" name="start" timeout="20s"/>
60de42
+          <op id="ip-192.168.24.6-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
60de42
+          <op id="ip-192.168.24.6-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <primitive class="ocf" id="ip-10.0.0.102" provider="heartbeat" type="IPaddr2">
60de42
+        <instance_attributes id="ip-10.0.0.102-instance_attributes">
60de42
+          <nvpair id="ip-10.0.0.102-instance_attributes-ip" name="ip" value="10.0.0.102"/>
60de42
+          <nvpair id="ip-10.0.0.102-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
60de42
+        </instance_attributes>
60de42
+        <meta_attributes id="ip-10.0.0.102-meta_attributes"/>
60de42
+        <operations>
60de42
+          <op id="ip-10.0.0.102-start-interval-0s" interval="0s" name="start" timeout="20s"/>
60de42
+          <op id="ip-10.0.0.102-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
60de42
+          <op id="ip-10.0.0.102-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <primitive class="ocf" id="ip-172.17.1.14" provider="heartbeat" type="IPaddr2">
60de42
+        <instance_attributes id="ip-172.17.1.14-instance_attributes">
60de42
+          <nvpair id="ip-172.17.1.14-instance_attributes-ip" name="ip" value="172.17.1.14"/>
60de42
+          <nvpair id="ip-172.17.1.14-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
60de42
+        </instance_attributes>
60de42
+        <meta_attributes id="ip-172.17.1.14-meta_attributes"/>
60de42
+        <operations>
60de42
+          <op id="ip-172.17.1.14-start-interval-0s" interval="0s" name="start" timeout="20s"/>
60de42
+          <op id="ip-172.17.1.14-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
60de42
+          <op id="ip-172.17.1.14-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <primitive class="ocf" id="ip-172.17.1.17" provider="heartbeat" type="IPaddr2">
60de42
+        <instance_attributes id="ip-172.17.1.17-instance_attributes">
60de42
+          <nvpair id="ip-172.17.1.17-instance_attributes-ip" name="ip" value="172.17.1.17"/>
60de42
+          <nvpair id="ip-172.17.1.17-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
60de42
+        </instance_attributes>
60de42
+        <meta_attributes id="ip-172.17.1.17-meta_attributes"/>
60de42
+        <operations>
60de42
+          <op id="ip-172.17.1.17-start-interval-0s" interval="0s" name="start" timeout="20s"/>
60de42
+          <op id="ip-172.17.1.17-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
60de42
+          <op id="ip-172.17.1.17-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <primitive class="ocf" id="ip-172.17.3.15" provider="heartbeat" type="IPaddr2">
60de42
+        <instance_attributes id="ip-172.17.3.15-instance_attributes">
60de42
+          <nvpair id="ip-172.17.3.15-instance_attributes-ip" name="ip" value="172.17.3.15"/>
60de42
+          <nvpair id="ip-172.17.3.15-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
60de42
+        </instance_attributes>
60de42
+        <meta_attributes id="ip-172.17.3.15-meta_attributes"/>
60de42
+        <operations>
60de42
+          <op id="ip-172.17.3.15-start-interval-0s" interval="0s" name="start" timeout="20s"/>
60de42
+          <op id="ip-172.17.3.15-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
60de42
+          <op id="ip-172.17.3.15-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <primitive class="ocf" id="ip-172.17.4.11" provider="heartbeat" type="IPaddr2">
60de42
+        <instance_attributes id="ip-172.17.4.11-instance_attributes">
60de42
+          <nvpair id="ip-172.17.4.11-instance_attributes-ip" name="ip" value="172.17.4.11"/>
60de42
+          <nvpair id="ip-172.17.4.11-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
60de42
+        </instance_attributes>
60de42
+        <meta_attributes id="ip-172.17.4.11-meta_attributes"/>
60de42
+        <operations>
60de42
+          <op id="ip-172.17.4.11-start-interval-0s" interval="0s" name="start" timeout="20s"/>
60de42
+          <op id="ip-172.17.4.11-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
60de42
+          <op id="ip-172.17.4.11-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <clone id="haproxy-clone">
60de42
+        <primitive class="systemd" id="haproxy" type="haproxy">
60de42
+          <instance_attributes id="haproxy-instance_attributes"/>
60de42
+          <meta_attributes id="haproxy-meta_attributes"/>
60de42
+          <operations>
60de42
+            <op id="haproxy-start-interval-0s" interval="0s" name="start" timeout="200s"/>
60de42
+            <op id="haproxy-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
60de42
+            <op id="haproxy-monitor-interval-60s" interval="60s" name="monitor"/>
60de42
+          </operations>
60de42
+        </primitive>
60de42
+        <meta_attributes id="haproxy-clone-meta_attributes"/>
60de42
+      </clone>
60de42
+      <primitive class="systemd" id="openstack-cinder-volume" type="openstack-cinder-volume">
60de42
+        <instance_attributes id="openstack-cinder-volume-instance_attributes"/>
60de42
+        <meta_attributes id="openstack-cinder-volume-meta_attributes"/>
60de42
+        <operations>
60de42
+          <op id="openstack-cinder-volume-start-interval-0s" interval="0s" name="start" timeout="200s"/>
60de42
+          <op id="openstack-cinder-volume-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
60de42
+          <op id="openstack-cinder-volume-monitor-interval-60s" interval="60s" name="monitor"/>
60de42
+        </operations>
60de42
+      </primitive>
60de42
+      <primitive class="stonith" id="stonith-fence_ipmilan-525400bbf613" type="fence_ipmilan">
60de42
+        <instance_attributes id="stonith-fence_ipmilan-525400bbf613-instance_attributes">
60de42
+          <nvpair id="stonith-fence_ipmilan-525400bbf613-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="controller-2"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400bbf613-instance_attributes-ipaddr" name="ipaddr" value="172.16.0.1"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400bbf613-instance_attributes-ipport" name="ipport" value="6237"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400bbf613-instance_attributes-passwd" name="passwd" value="****"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400bbf613-instance_attributes-lanplus" name="lanplus" value="true"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400bbf613-instance_attributes-login" name="login" value="admin"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400bbf613-instance_attributes-action" name="action" value="reboot"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400bbf613-instance_attributes-privlvl" name="privlvl" value="administrator"/>
60de42
+        </instance_attributes>
60de42
+        <operations>
60de42
+          <op id="stonith-fence_ipmilan-525400bbf613-monitor-interval-60s" interval="60s" name="monitor"/>
60de42
+        </operations>
60de42
+        <meta_attributes id="stonith-fence_ipmilan-525400bbf613-meta_attributes"/>
60de42
+      </primitive>
60de42
+      <primitive class="stonith" id="stonith-fence_ipmilan-525400b4f6bd" type="fence_ipmilan">
60de42
+        <instance_attributes id="stonith-fence_ipmilan-525400b4f6bd-instance_attributes">
60de42
+          <nvpair id="stonith-fence_ipmilan-525400b4f6bd-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="controller-1"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400b4f6bd-instance_attributes-ipaddr" name="ipaddr" value="172.16.0.1"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400b4f6bd-instance_attributes-ipport" name="ipport" value="6236"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400b4f6bd-instance_attributes-passwd" name="passwd" value="****"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400b4f6bd-instance_attributes-lanplus" name="lanplus" value="true"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400b4f6bd-instance_attributes-login" name="login" value="admin"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400b4f6bd-instance_attributes-action" name="action" value="reboot"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-525400b4f6bd-instance_attributes-privlvl" name="privlvl" value="administrator"/>
60de42
+        </instance_attributes>
60de42
+        <operations>
60de42
+          <op id="stonith-fence_ipmilan-525400b4f6bd-monitor-interval-60s" interval="60s" name="monitor"/>
60de42
+        </operations>
60de42
+        <meta_attributes id="stonith-fence_ipmilan-525400b4f6bd-meta_attributes"/>
60de42
+      </primitive>
60de42
+      <primitive class="stonith" id="stonith-fence_ipmilan-5254005bdbb5" type="fence_ipmilan">
60de42
+        <instance_attributes id="stonith-fence_ipmilan-5254005bdbb5-instance_attributes">
60de42
+          <nvpair id="stonith-fence_ipmilan-5254005bdbb5-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="controller-0"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-5254005bdbb5-instance_attributes-ipaddr" name="ipaddr" value="172.16.0.1"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-5254005bdbb5-instance_attributes-ipport" name="ipport" value="6235"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-5254005bdbb5-instance_attributes-passwd" name="passwd" value="****"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-5254005bdbb5-instance_attributes-lanplus" name="lanplus" value="true"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-5254005bdbb5-instance_attributes-login" name="login" value="admin"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-5254005bdbb5-instance_attributes-action" name="action" value="reboot"/>
60de42
+          <nvpair id="stonith-fence_ipmilan-5254005bdbb5-instance_attributes-privlvl" name="privlvl" value="administrator"/>
60de42
+        </instance_attributes>
60de42
+        <operations>
60de42
+          <op id="stonith-fence_ipmilan-5254005bdbb5-monitor-interval-60s" interval="60s" name="monitor"/>
60de42
+        </operations>
60de42
+        <meta_attributes id="stonith-fence_ipmilan-5254005bdbb5-meta_attributes"/>
60de42
+      </primitive>
60de42
+    </resources>
60de42
+    <constraints>
60de42
+      <rsc_location id="location-rabbitmq-clone" resource-discovery="exclusive" rsc="rabbitmq-clone">
60de42
+        <rule id="location-rabbitmq-clone-rule" score="0">
60de42
+          <expression attribute="rabbitmq-role" id="location-rabbitmq-clone-rule-expr" operation="eq" value="true"/>
60de42
+        </rule>
60de42
+      </rsc_location>
60de42
+      <rsc_location id="location-galera-master" resource-discovery="exclusive" rsc="galera-master">
60de42
+        <rule id="location-galera-master-rule" score="0">
60de42
+          <expression attribute="galera-role" id="location-galera-master-rule-expr" operation="eq" value="true"/>
60de42
+        </rule>
60de42
+      </rsc_location>
60de42
+      <rsc_location id="location-redis-master" resource-discovery="exclusive" rsc="redis-master">
60de42
+        <rule id="location-redis-master-rule" score="0">
60de42
+          <expression attribute="redis-role" id="location-redis-master-rule-expr" operation="eq" value="true"/>
60de42
+        </rule>
60de42
+      </rsc_location>
60de42
+      <rsc_location id="location-ip-192.168.24.6" resource-discovery="exclusive" rsc="ip-192.168.24.6">
60de42
+        <rule id="location-ip-192.168.24.6-rule" score="0">
60de42
+          <expression attribute="haproxy-role" id="location-ip-192.168.24.6-rule-expr" operation="eq" value="true"/>
60de42
+        </rule>
60de42
+      </rsc_location>
60de42
+      <rsc_location id="location-ip-10.0.0.102" resource-discovery="exclusive" rsc="ip-10.0.0.102">
60de42
+        <rule id="location-ip-10.0.0.102-rule" score="0">
60de42
+          <expression attribute="haproxy-role" id="location-ip-10.0.0.102-rule-expr" operation="eq" value="true"/>
60de42
+        </rule>
60de42
+      </rsc_location>
60de42
+      <rsc_location id="location-ip-172.17.1.14" resource-discovery="exclusive" rsc="ip-172.17.1.14">
60de42
+        <rule id="location-ip-172.17.1.14-rule" score="0">
60de42
+          <expression attribute="haproxy-role" id="location-ip-172.17.1.14-rule-expr" operation="eq" value="true"/>
60de42
+        </rule>
60de42
+      </rsc_location>
60de42
+      <rsc_location id="location-ip-172.17.1.17" resource-discovery="exclusive" rsc="ip-172.17.1.17">
60de42
+        <rule id="location-ip-172.17.1.17-rule" score="0">
60de42
+          <expression attribute="haproxy-role" id="location-ip-172.17.1.17-rule-expr" operation="eq" value="true"/>
60de42
+        </rule>
60de42
+      </rsc_location>
60de42
+      <rsc_location id="location-ip-172.17.3.15" resource-discovery="exclusive" rsc="ip-172.17.3.15">
60de42
+        <rule id="location-ip-172.17.3.15-rule" score="0">
60de42
+          <expression attribute="haproxy-role" id="location-ip-172.17.3.15-rule-expr" operation="eq" value="true"/>
60de42
+        </rule>
60de42
+      </rsc_location>
60de42
+      <rsc_location id="location-ip-172.17.4.11" resource-discovery="exclusive" rsc="ip-172.17.4.11">
60de42
+        <rule id="location-ip-172.17.4.11-rule" score="0">
60de42
+          <expression attribute="haproxy-role" id="location-ip-172.17.4.11-rule-expr" operation="eq" value="true"/>
60de42
+        </rule>
60de42
+      </rsc_location>
60de42
+      <rsc_location id="location-haproxy-clone" resource-discovery="exclusive" rsc="haproxy-clone">
60de42
+        <rule id="location-haproxy-clone-rule" score="0">
60de42
+          <expression attribute="haproxy-role" id="location-haproxy-clone-rule-expr" operation="eq" value="true"/>
60de42
+        </rule>
60de42
+      </rsc_location>
60de42
+      <rsc_order first="ip-192.168.24.6" first-action="start" id="order-ip-192.168.24.6-haproxy-clone-Optional" kind="Optional" then="haproxy-clone" then-action="start"/>
60de42
+      <rsc_colocation id="colocation-ip-192.168.24.6-haproxy-clone-INFINITY" rsc="ip-192.168.24.6" score="INFINITY" with-rsc="haproxy-clone"/>
60de42
+      <rsc_order first="ip-10.0.0.102" first-action="start" id="order-ip-10.0.0.102-haproxy-clone-Optional" kind="Optional" then="haproxy-clone" then-action="start"/>
60de42
+      <rsc_colocation id="colocation-ip-10.0.0.102-haproxy-clone-INFINITY" rsc="ip-10.0.0.102" score="INFINITY" with-rsc="haproxy-clone"/>
60de42
+      <rsc_order first="ip-172.17.1.14" first-action="start" id="order-ip-172.17.1.14-haproxy-clone-Optional" kind="Optional" then="haproxy-clone" then-action="start"/>
60de42
+      <rsc_colocation id="colocation-ip-172.17.1.14-haproxy-clone-INFINITY" rsc="ip-172.17.1.14" score="INFINITY" with-rsc="haproxy-clone"/>
60de42
+      <rsc_order first="ip-172.17.1.17" first-action="start" id="order-ip-172.17.1.17-haproxy-clone-Optional" kind="Optional" then="haproxy-clone" then-action="start"/>
60de42
+      <rsc_colocation id="colocation-ip-172.17.1.17-haproxy-clone-INFINITY" rsc="ip-172.17.1.17" score="INFINITY" with-rsc="haproxy-clone"/>
60de42
+      <rsc_order first="ip-172.17.3.15" first-action="start" id="order-ip-172.17.3.15-haproxy-clone-Optional" kind="Optional" then="haproxy-clone" then-action="start"/>
60de42
+      <rsc_colocation id="colocation-ip-172.17.3.15-haproxy-clone-INFINITY" rsc="ip-172.17.3.15" score="INFINITY" with-rsc="haproxy-clone"/>
60de42
+      <rsc_order first="ip-172.17.4.11" first-action="start" id="order-ip-172.17.4.11-haproxy-clone-Optional" kind="Optional" then="haproxy-clone" then-action="start"/>
60de42
+      <rsc_colocation id="colocation-ip-172.17.4.11-haproxy-clone-INFINITY" rsc="ip-172.17.4.11" score="INFINITY" with-rsc="haproxy-clone"/>
60de42
+      <rsc_location id="location-openstack-cinder-volume" resource-discovery="exclusive" rsc="openstack-cinder-volume">
60de42
+        <rule id="location-openstack-cinder-volume-rule" score="0">
60de42
+          <expression attribute="cinder-volume-role" id="location-openstack-cinder-volume-rule-expr" operation="eq" value="true"/>
60de42
+        </rule>
60de42
+      </rsc_location>
60de42
+      <rsc_location id="location-stonith-fence_ipmilan-525400bbf613-controller-2--INFINITY" node="controller-2" rsc="stonith-fence_ipmilan-525400bbf613" score="-INFINITY"/>
60de42
+      <rsc_location id="location-stonith-fence_ipmilan-525400b4f6bd-controller-1--INFINITY" node="controller-1" rsc="stonith-fence_ipmilan-525400b4f6bd" score="-INFINITY"/>
60de42
+      <rsc_location id="location-stonith-fence_ipmilan-5254005bdbb5-controller-0--INFINITY" node="controller-0" rsc="stonith-fence_ipmilan-5254005bdbb5" score="-INFINITY"/>
60de42
+    </constraints>
60de42
+    <rsc_defaults>
60de42
+      <meta_attributes id="rsc_defaults-options">
60de42
+        <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
60de42
+      </meta_attributes>
60de42
+    </rsc_defaults>
60de42
+  </configuration>
60de42
+  <status>
60de42
+    <node_state id="2" uname="controller-1" in_ccm="false" crmd="offline" crm-debug-origin="post_cache_update" join="member" expected="member">
60de42
+      <lrm id="2">
60de42
+        <lrm_resources>
60de42
+          <lrm_resource id="ip-10.0.0.102" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-10.0.0.102_last_0" operation_key="ip-10.0.0.102_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="37:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:7;37:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="20" rc-code="7" op-status="0" interval="0" last-run="1493817888" last-rc-change="1493817888" exec-time="44" queue-time="0" op-digest="194c484ea9d1eb9955c0cfdaf53ccdae"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="haproxy" type="haproxy" class="systemd">
60de42
+            <lrm_rsc_op id="haproxy_last_0" operation_key="haproxy_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="138:5:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;138:5:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="46" rc-code="0" op-status="0" interval="0" last-run="1493817889" last-rc-change="1493817889" exec-time="2088" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+            <lrm_rsc_op id="haproxy_monitor_60000" operation_key="haproxy_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="119:6:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;119:6:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="63" rc-code="0" op-status="0" interval="60000" last-rc-change="1493817892" exec-time="1" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="openstack-cinder-volume" type="openstack-cinder-volume" class="systemd">
60de42
+            <lrm_rsc_op id="openstack-cinder-volume_last_0" operation_key="openstack-cinder-volume_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="43:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:7;43:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="45" rc-code="7" op-status="0" interval="0" last-run="1493817889" last-rc-change="1493817889" exec-time="3" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.4.11" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.4.11_last_0" operation_key="ip-172.17.4.11_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="118:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;118:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="70" rc-code="0" op-status="0" interval="0" last-run="1493817963" last-rc-change="1493817963" exec-time="58" queue-time="0" op-digest="fe27c3edbb73480aff571d34c88b21a7"/>
60de42
+            <lrm_rsc_op id="ip-172.17.4.11_monitor_10000" operation_key="ip-172.17.4.11_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="119:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;119:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="71" rc-code="0" op-status="0" interval="10000" last-rc-change="1493817964" exec-time="38" queue-time="0" op-digest="392445a354e2da75dcd4cbe6eee1268b"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="messaging-0" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="messaging-0_last_0" operation_key="messaging-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="29:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:7;29:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1493817888" last-rc-change="1493817888" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="messaging-1" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="messaging-1_last_0" operation_key="messaging-1_migrate_from_0" operation="migrate_from" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="38:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;38:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="7" rc-code="0" op-status="0" interval="0" last-run="1493817961" last-rc-change="1493817961" exec-time="0" queue-time="0" migrate_source="controller-2" migrate_target="controller-1" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+            <lrm_rsc_op id="messaging-1_monitor_20000" operation_key="messaging-1_monitor_20000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="36:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;36:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="10" rc-code="0" op-status="0" interval="20000" last-rc-change="1493817963" exec-time="0" queue-time="0" op-digest="6e5bb737f46c381d8a46fb4162afd9e0"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="messaging-2" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="messaging-2_last_0" operation_key="messaging-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="31:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:7;31:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1493817888" last-rc-change="1493817888" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-192.168.24.6" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-192.168.24.6_last_0" operation_key="ip-192.168.24.6_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="36:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:7;36:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1493817888" last-rc-change="1493817888" exec-time="49" queue-time="0" op-digest="22692fc1751093fc15ab6c0d90fafe22"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.3.15" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.3.15_last_0" operation_key="ip-172.17.3.15_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="40:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:7;40:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="32" rc-code="7" op-status="0" interval="0" last-run="1493817888" last-rc-change="1493817888" exec-time="45" queue-time="0" op-digest="27e0700bb641af6a5b0fea09d0ea2482"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.1.14" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.1.14_last_0" operation_key="ip-172.17.1.14_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="110:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;110:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="69" rc-code="0" op-status="0" interval="0" last-run="1493817963" last-rc-change="1493817963" exec-time="60" queue-time="0" op-digest="1fab2783a8d283e33a945588908e98a4"/>
60de42
+            <lrm_rsc_op id="ip-172.17.1.14_monitor_10000" operation_key="ip-172.17.1.14_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="111:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;111:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="72" rc-code="0" op-status="0" interval="10000" last-rc-change="1493817964" exec-time="36" queue-time="0" op-digest="19c32490a75539eb9cf2ca18727e305e"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.1.17" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.1.17_last_0" operation_key="ip-172.17.1.17_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="113:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;113:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="68" rc-code="0" op-status="0" interval="0" last-run="1493817963" last-rc-change="1493817963" exec-time="62" queue-time="0" op-digest="edc8fdae4cc326c15a779f36f28eb3f8"/>
60de42
+            <lrm_rsc_op id="ip-172.17.1.17_monitor_10000" operation_key="ip-172.17.1.17_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="114:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;114:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="73" rc-code="0" op-status="0" interval="10000" last-rc-change="1493817964" exec-time="36" queue-time="0" op-digest="637be44014a8de2a8162cb2b062f6955"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="stonith-fence_ipmilan-525400bbf613" type="fence_ipmilan" class="stonith">
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-525400bbf613_last_0" operation_key="stonith-fence_ipmilan-525400bbf613_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="44:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:7;44:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="50" rc-code="7" op-status="0" interval="0" last-run="1493817889" last-rc-change="1493817889" exec-time="3" queue-time="0" op-digest="a1b3d562a7e722cbf686e1c2750888a3" op-secure-params=" passwd " op-secure-digest="2d0324390caa209c81283f07749d6dae"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="stonith-fence_ipmilan-5254005bdbb5" type="fence_ipmilan" class="stonith">
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-5254005bdbb5_last_0" operation_key="stonith-fence_ipmilan-5254005bdbb5_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="148:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;148:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="65" rc-code="0" op-status="0" interval="0" last-run="1493817961" last-rc-change="1493817961" exec-time="79" queue-time="0" op-digest="25087bde0e53bf4f4e4efb2b0bd9d0de" op-secure-params=" passwd " op-secure-digest="ecf31bd4bca13bb934be683ab04ee8cf"/>
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-5254005bdbb5_monitor_60000" operation_key="stonith-fence_ipmilan-5254005bdbb5_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="149:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;149:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="66" rc-code="0" op-status="0" interval="60000" last-rc-change="1493817963" exec-time="80" queue-time="0" op-digest="e8a37549d88ca88c76d297b973759a8d" op-secure-params=" passwd " op-secure-digest="ecf31bd4bca13bb934be683ab04ee8cf"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="redis_last_0" operation_key="redis_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="95:5:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;95:5:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="55" rc-code="0" op-status="0" interval="0" last-run="1493817890" last-rc-change="1493817890" exec-time="2553" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
60de42
+            <lrm_rsc_op id="redis_monitor_45000" operation_key="redis_monitor_45000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="73:6:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;73:6:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="62" rc-code="0" op-status="0" interval="45000" last-rc-change="1493817892" exec-time="124" queue-time="135" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
60de42
+            <lrm_rsc_op id="redis_monitor_60000" operation_key="redis_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="74:6:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;74:6:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="61" rc-code="0" op-status="0" interval="60000" last-rc-change="1493817892" exec-time="135" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="galera-0" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="galera-0_last_0" operation_key="galera-0_migrate_from_0" operation="migrate_from" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="48:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;48:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="8" rc-code="0" op-status="0" interval="0" last-run="1493817961" last-rc-change="1493817961" exec-time="0" queue-time="0" migrate_source="controller-2" migrate_target="controller-1" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+            <lrm_rsc_op id="galera-0_monitor_20000" operation_key="galera-0_monitor_20000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="38:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;38:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="11" rc-code="0" op-status="0" interval="20000" last-rc-change="1493817963" exec-time="0" queue-time="0" op-digest="6e5bb737f46c381d8a46fb4162afd9e0"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="galera-1" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="galera-1_last_0" operation_key="galera-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="33:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:7;33:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1493817888" last-rc-change="1493817888" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="galera-2" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="galera-2_last_0" operation_key="galera-2_migrate_from_0" operation="migrate_from" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="55:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;55:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="9" rc-code="0" op-status="0" interval="0" last-run="1493817961" last-rc-change="1493817961" exec-time="0" queue-time="0" migrate_source="controller-2" migrate_target="controller-1" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+            <lrm_rsc_op id="galera-2_monitor_20000" operation_key="galera-2_monitor_20000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="44:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;44:9:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="12" rc-code="0" op-status="0" interval="20000" last-rc-change="1493817963" exec-time="0" queue-time="0" op-digest="6e5bb737f46c381d8a46fb4162afd9e0"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="stonith-fence_ipmilan-525400b4f6bd" type="fence_ipmilan" class="stonith">
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-525400b4f6bd_last_0" operation_key="stonith-fence_ipmilan-525400b4f6bd_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="45:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:7;45:5:7:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-1" call-id="54" rc-code="7" op-status="0" interval="0" last-run="1493817889" last-rc-change="1493817889" exec-time="0" queue-time="0" op-digest="190a5d7184e8c0736ea8d50b25d1896b" op-secure-params=" passwd " op-secure-digest="e710ff47e4fa69f83f5f46f9de9570b6"/>
60de42
+          </lrm_resource>
60de42
+        </lrm_resources>
60de42
+      </lrm>
60de42
+    </node_state>
60de42
+    <node_state id="1" uname="controller-0" crmd="online" crm-debug-origin="post_cache_update" in_ccm="true" join="member" expected="member">
60de42
+      <transient_attributes id="1">
60de42
+        <instance_attributes id="status-1">
60de42
+          <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
60de42
+          <nvpair id="status-1-master-redis" name="master-redis" value="1"/>
60de42
+        </instance_attributes>
60de42
+      </transient_attributes>
60de42
+      <lrm id="1">
60de42
+        <lrm_resources>
60de42
+          <lrm_resource id="ip-10.0.0.102" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-10.0.0.102_last_0" operation_key="ip-10.0.0.102_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="119:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;119:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="143" rc-code="0" op-status="0" interval="0" last-run="1493817834" last-rc-change="1493817834" exec-time="497" queue-time="0" op-digest="194c484ea9d1eb9955c0cfdaf53ccdae"/>
60de42
+            <lrm_rsc_op id="ip-10.0.0.102_monitor_10000" operation_key="ip-10.0.0.102_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="120:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;120:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="147" rc-code="0" op-status="0" interval="10000" last-rc-change="1493817835" exec-time="48" queue-time="0" op-digest="83daeb09ea7f361d0c80eb34aaf106f9"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="haproxy" type="haproxy" class="systemd">
60de42
+            <lrm_rsc_op id="haproxy_last_failure_0" operation_key="haproxy_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="15:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;15:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="114" rc-code="0" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="3" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+            <lrm_rsc_op id="haproxy_last_0" operation_key="haproxy_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="15:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;15:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="114" rc-code="0" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="3" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+            <lrm_rsc_op id="haproxy_monitor_60000" operation_key="haproxy_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="113:17:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;113:17:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="133" rc-code="0" op-status="0" interval="60000" last-rc-change="1493817765" exec-time="4" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="openstack-cinder-volume" type="openstack-cinder-volume" class="systemd">
60de42
+            <lrm_rsc_op id="openstack-cinder-volume_last_0" operation_key="openstack-cinder-volume_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="119:23:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;119:23:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="150" rc-code="0" op-status="0" interval="0" last-run="1493817868" last-rc-change="1493817868" exec-time="2105" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+            <lrm_rsc_op id="openstack-cinder-volume_monitor_60000" operation_key="openstack-cinder-volume_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="145:1:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;145:1:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-0" call-id="154" rc-code="0" op-status="0" interval="60000" last-rc-change="1493817878" exec-time="3" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.4.11" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.4.11_last_0" operation_key="ip-172.17.4.11_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="14:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:7;14:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="109" rc-code="7" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="66" queue-time="0" op-digest="fe27c3edbb73480aff571d34c88b21a7"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="messaging-0" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="messaging-0_last_0" operation_key="messaging-0_migrate_from_0" operation="migrate_from" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="36:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;36:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1493817832" last-rc-change="1493817832" exec-time="0" queue-time="0" migrate_source="controller-1" migrate_target="controller-0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+            <lrm_rsc_op id="messaging-0_monitor_20000" operation_key="messaging-0_monitor_20000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="34:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;34:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="15" rc-code="0" op-status="0" interval="20000" last-rc-change="1493817832" exec-time="0" queue-time="0" op-digest="6e5bb737f46c381d8a46fb4162afd9e0"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="messaging-1" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="messaging-1_last_0" operation_key="messaging-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="3:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:7;3:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="8" rc-code="7" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="messaging-2" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="messaging-2_last_0" operation_key="messaging-2_migrate_from_0" operation="migrate_from" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="43:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;43:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-0" call-id="17" rc-code="0" op-status="0" interval="0" last-run="1493817962" last-rc-change="1493817962" exec-time="0" queue-time="0" migrate_source="controller-2" migrate_target="controller-0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+            <lrm_rsc_op id="messaging-2_monitor_20000" operation_key="messaging-2_monitor_20000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="41:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:0;41:8:0:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-0" call-id="18" rc-code="0" op-status="0" interval="20000" last-rc-change="1493817963" exec-time="0" queue-time="0" op-digest="6e5bb737f46c381d8a46fb4162afd9e0"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.1.14" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.1.14_last_0" operation_key="ip-172.17.1.14_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="11:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:7;11:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="97" rc-code="7" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="60" queue-time="0" op-digest="1fab2783a8d283e33a945588908e98a4"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="stonith-fence_ipmilan-525400bbf613" type="fence_ipmilan" class="stonith">
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-525400bbf613_last_0" operation_key="stonith-fence_ipmilan-525400bbf613_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="124:17:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;124:17:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="134" rc-code="0" op-status="0" interval="0" last-run="1493817765" last-rc-change="1493817765" exec-time="83" queue-time="0" op-digest="a1b3d562a7e722cbf686e1c2750888a3" op-secure-params=" passwd " op-secure-digest="2d0324390caa209c81283f07749d6dae"/>
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-525400bbf613_monitor_60000" operation_key="stonith-fence_ipmilan-525400bbf613_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="129:18:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;129:18:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="136" rc-code="0" op-status="0" interval="60000" last-rc-change="1493817768" exec-time="68" queue-time="0" op-digest="31be355d25ac30183b5d304f3275ec16" op-secure-params=" passwd " op-secure-digest="2d0324390caa209c81283f07749d6dae"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.1.17" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.1.17_last_0" operation_key="ip-172.17.1.17_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="12:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:7;12:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="101" rc-code="7" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="68" queue-time="0" op-digest="edc8fdae4cc326c15a779f36f28eb3f8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="stonith-fence_ipmilan-5254005bdbb5" type="fence_ipmilan" class="stonith">
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-5254005bdbb5_last_0" operation_key="stonith-fence_ipmilan-5254005bdbb5_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="19:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:7;19:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="130" rc-code="7" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="0" queue-time="0" op-digest="25087bde0e53bf4f4e4efb2b0bd9d0de" op-secure-params=" passwd " op-secure-digest="ecf31bd4bca13bb934be683ab04ee8cf"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.3.15" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.3.15_last_0" operation_key="ip-172.17.3.15_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="126:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;126:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="141" rc-code="0" op-status="0" interval="0" last-run="1493817834" last-rc-change="1493817834" exec-time="66" queue-time="0" op-digest="27e0700bb641af6a5b0fea09d0ea2482"/>
60de42
+            <lrm_rsc_op id="ip-172.17.3.15_monitor_10000" operation_key="ip-172.17.3.15_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="127:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;127:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="144" rc-code="0" op-status="0" interval="10000" last-rc-change="1493817834" exec-time="428" queue-time="0" op-digest="9f5ae0b4f77392ad67e6a430d3259ec4"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-192.168.24.6" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-192.168.24.6_last_0" operation_key="ip-192.168.24.6_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="116:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;116:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="142" rc-code="0" op-status="0" interval="0" last-run="1493817834" last-rc-change="1493817834" exec-time="498" queue-time="0" op-digest="22692fc1751093fc15ab6c0d90fafe22"/>
60de42
+            <lrm_rsc_op id="ip-192.168.24.6_monitor_10000" operation_key="ip-192.168.24.6_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="117:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;117:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="146" rc-code="0" op-status="0" interval="10000" last-rc-change="1493817835" exec-time="38" queue-time="0" op-digest="53892658d5fe65d671f4c791888c8c7c"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="redis_last_failure_0" operation_key="redis_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="8:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;8:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="85" rc-code="0" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="171" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
60de42
+            <lrm_rsc_op id="redis_last_0" operation_key="redis_promote_0" operation="promote" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="71:23:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;71:23:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="152" rc-code="0" op-status="0" interval="0" last-run="1493817868" last-rc-change="1493817868" exec-time="255" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
60de42
+            <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="74:4:8:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" transition-magic="0:8;74:4:8:5c19b904-a6d8-4a4f-9b6d-9ab13f4f6b57" on_node="controller-0" call-id="159" rc-code="8" op-status="0" interval="20000" last-rc-change="1493817884" exec-time="147" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="galera-0" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="galera-0_last_0" operation_key="galera-0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="5:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:7;5:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="10" rc-code="7" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="galera-1" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="galera-1_last_0" operation_key="galera-1_migrate_from_0" operation="migrate_from" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="50:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;50:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1493817832" last-rc-change="1493817832" exec-time="0" queue-time="0" migrate_source="controller-1" migrate_target="controller-0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+            <lrm_rsc_op id="galera-1_monitor_20000" operation_key="galera-1_monitor_20000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="48:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;48:22:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="16" rc-code="0" op-status="0" interval="20000" last-rc-change="1493817832" exec-time="0" queue-time="0" op-digest="6e5bb737f46c381d8a46fb4162afd9e0"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="galera-2" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="galera-2_last_0" operation_key="galera-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="7:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:7;7:16:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="12" rc-code="7" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="stonith-fence_ipmilan-525400b4f6bd" type="fence_ipmilan" class="stonith">
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-525400b4f6bd_last_0" operation_key="stonith-fence_ipmilan-525400b4f6bd_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="126:17:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;126:17:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="135" rc-code="0" op-status="0" interval="0" last-run="1493817765" last-rc-change="1493817765" exec-time="80" queue-time="0" op-digest="190a5d7184e8c0736ea8d50b25d1896b" op-secure-params=" passwd " op-secure-digest="e710ff47e4fa69f83f5f46f9de9570b6"/>
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-525400b4f6bd_monitor_60000" operation_key="stonith-fence_ipmilan-525400b4f6bd_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="132:18:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;132:18:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-0" call-id="137" rc-code="0" op-status="0" interval="60000" last-rc-change="1493817768" exec-time="69" queue-time="0" op-digest="08b6ad9edc64306555c732c45bcf4995" op-secure-params=" passwd " op-secure-digest="e710ff47e4fa69f83f5f46f9de9570b6"/>
60de42
+          </lrm_resource>
60de42
+        </lrm_resources>
60de42
+      </lrm>
60de42
+    </node_state>
60de42
+    <node_state id="galera-1" remote_node="true" uname="galera-1" in_ccm="true" crm-debug-origin="post_cache_update">
60de42
+      <transient_attributes id="galera-1">
60de42
+        <instance_attributes id="status-galera-1">
60de42
+          <nvpair id="status-galera-1-last-failure-galera" name="last-failure-galera" value="1493336863"/>
60de42
+          <nvpair id="status-galera-1-master-galera" name="master-galera" value="100"/>
60de42
+        </instance_attributes>
60de42
+      </transient_attributes>
60de42
+      <lrm id="galera-1">
60de42
+        <lrm_resources>
60de42
+          <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="galera_last_failure_0" operation_key="galera_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="3:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:8;3:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-1" call-id="34745" rc-code="8" op-status="0" interval="0" last-run="1493817765" last-rc-change="1493817765" exec-time="463" queue-time="0" op-digest="e773372fde8a6c242c1993820114d5c3"/>
60de42
+            <lrm_rsc_op id="galera_last_0" operation_key="galera_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="3:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:8;3:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-1" call-id="34745" rc-code="8" op-status="0" interval="0" last-run="1493817765" last-rc-change="1493817765" exec-time="463" queue-time="0" op-digest="e773372fde8a6c242c1993820114d5c3" op-secure-params=" user " op-secure-digest="e773372fde8a6c242c1993820114d5c3"/>
60de42
+            <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="52:18:8:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:8;52:18:8:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-1" call-id="34756" rc-code="8" op-status="0" interval="10000" last-rc-change="1493817768" exec-time="65" queue-time="0" op-digest="20ec44e3280211739f6ba9523159629e" op-secure-params=" user " op-secure-digest="e773372fde8a6c242c1993820114d5c3"/>
60de42
+          </lrm_resource>
60de42
+        </lrm_resources>
60de42
+      </lrm>
60de42
+    </node_state>
60de42
+    <node_state id="galera-2" remote_node="true" uname="galera-2" crm-debug-origin="post_cache_update" in_ccm="true">
60de42
+      <transient_attributes id="galera-2">
60de42
+        <instance_attributes id="status-galera-2">
60de42
+          <nvpair id="status-galera-2-last-failure-galera" name="last-failure-galera" value="1493172798"/>
60de42
+          <nvpair id="status-galera-2-master-galera" name="master-galera" value="100"/>
60de42
+        </instance_attributes>
60de42
+      </transient_attributes>
60de42
+      <lrm id="galera-2">
60de42
+        <lrm_resources>
60de42
+          <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="galera_last_failure_0" operation_key="galera_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="4:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:8;4:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-2" call-id="34779" rc-code="8" op-status="0" interval="0" last-run="1493817766" last-rc-change="1493817766" exec-time="1419" queue-time="0" op-digest="e773372fde8a6c242c1993820114d5c3"/>
60de42
+            <lrm_rsc_op id="galera_last_0" operation_key="galera_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="4:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:8;4:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-2" call-id="34779" rc-code="8" op-status="0" interval="0" last-run="1493817766" last-rc-change="1493817766" exec-time="1419" queue-time="0" op-digest="e773372fde8a6c242c1993820114d5c3" op-secure-params=" user " op-secure-digest="e773372fde8a6c242c1993820114d5c3"/>
60de42
+            <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="57:18:8:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:8;57:18:8:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-2" call-id="34789" rc-code="8" op-status="0" interval="10000" last-rc-change="1493817768" exec-time="61" queue-time="0" op-digest="20ec44e3280211739f6ba9523159629e" op-secure-params=" user " op-secure-digest="e773372fde8a6c242c1993820114d5c3"/>
60de42
+          </lrm_resource>
60de42
+        </lrm_resources>
60de42
+      </lrm>
60de42
+    </node_state>
60de42
+    <node_state id="galera-0" remote_node="true" uname="galera-0" crm-debug-origin="post_cache_update" in_ccm="true">
60de42
+      <transient_attributes id="galera-0">
60de42
+        <instance_attributes id="status-galera-0">
60de42
+          <nvpair id="status-galera-0-last-failure-galera" name="last-failure-galera" value="1493172797"/>
60de42
+          <nvpair id="status-galera-0-master-galera" name="master-galera" value="100"/>
60de42
+        </instance_attributes>
60de42
+      </transient_attributes>
60de42
+      <lrm id="galera-0">
60de42
+        <lrm_resources>
60de42
+          <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="galera_last_failure_0" operation_key="galera_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="2:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:8;2:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-2" call-id="35583" rc-code="8" op-status="0" interval="0" last-run="1493817765" last-rc-change="1493817765" exec-time="1739" queue-time="0" op-digest="e773372fde8a6c242c1993820114d5c3"/>
60de42
+            <lrm_rsc_op id="galera_last_0" operation_key="galera_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="2:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:8;2:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-2" call-id="35583" rc-code="8" op-status="0" interval="0" last-run="1493817765" last-rc-change="1493817765" exec-time="1739" queue-time="0" op-digest="e773372fde8a6c242c1993820114d5c3" op-secure-params=" user " op-secure-digest="e773372fde8a6c242c1993820114d5c3"/>
60de42
+            <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="62:18:8:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:8;62:18:8:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-2" call-id="35593" rc-code="8" op-status="0" interval="10000" last-rc-change="1493817768" exec-time="60" queue-time="0" op-digest="20ec44e3280211739f6ba9523159629e" op-secure-params=" user " op-secure-digest="e773372fde8a6c242c1993820114d5c3"/>
60de42
+          </lrm_resource>
60de42
+        </lrm_resources>
60de42
+      </lrm>
60de42
+    </node_state>
60de42
+    <node_state id="3" uname="controller-2" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" join="member" expected="member">
60de42
+      <lrm id="3">
60de42
+        <lrm_resources>
60de42
+          <lrm_resource id="messaging-0" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="messaging-0_last_0" operation_key="messaging-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="29:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;29:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="messaging-1" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="messaging-1_last_0" operation_key="messaging-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="30:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;30:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="messaging-2" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="messaging-2_last_0" operation_key="messaging-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="31:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;31:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="galera-0" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="galera-0_last_0" operation_key="galera-0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="32:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;32:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="galera-1" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="galera-1_last_0" operation_key="galera-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="33:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;33:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="galera-2" type="remote" class="ocf" provider="pacemaker">
60de42
+            <lrm_rsc_op id="galera-2_last_0" operation_key="galera-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="34:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;34:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="6" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="0" queue-time="0" op-digest="a48beba1b11f09d6b1c15da8db5bb0a2" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="haproxy" type="haproxy" class="systemd">
60de42
+            <lrm_rsc_op id="haproxy_last_0" operation_key="haproxy_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="138:1:0:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:0;138:1:0:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="46" rc-code="0" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="2103" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+            <lrm_rsc_op id="haproxy_monitor_60000" operation_key="haproxy_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="123:2:0:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:0;123:2:0:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="63" rc-code="0" op-status="0" interval="60000" last-rc-change="1493817975" exec-time="3" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="openstack-cinder-volume" type="openstack-cinder-volume" class="systemd">
60de42
+            <lrm_rsc_op id="openstack-cinder-volume_last_0" operation_key="openstack-cinder-volume_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="43:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;43:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="45" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="2" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-192.168.24.6" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-192.168.24.6_last_0" operation_key="ip-192.168.24.6_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="36:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;36:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="42" queue-time="0" op-digest="22692fc1751093fc15ab6c0d90fafe22"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.1.14" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.1.14_last_0" operation_key="ip-172.17.1.14_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="38:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;38:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="24" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="52" queue-time="0" op-digest="1fab2783a8d283e33a945588908e98a4"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-10.0.0.102" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-10.0.0.102_last_0" operation_key="ip-10.0.0.102_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="37:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;37:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="20" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="59" queue-time="0" op-digest="194c484ea9d1eb9955c0cfdaf53ccdae"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.3.15" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.3.15_last_0" operation_key="ip-172.17.3.15_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="40:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;40:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="32" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="52" queue-time="0" op-digest="27e0700bb641af6a5b0fea09d0ea2482"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.4.11" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.4.11_last_0" operation_key="ip-172.17.4.11_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="41:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;41:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="36" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="50" queue-time="0" op-digest="fe27c3edbb73480aff571d34c88b21a7"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="ip-172.17.1.17" type="IPaddr2" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="ip-172.17.1.17_last_0" operation_key="ip-172.17.1.17_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="39:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;39:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="28" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="55" queue-time="0" op-digest="edc8fdae4cc326c15a779f36f28eb3f8"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="stonith-fence_ipmilan-525400bbf613" type="fence_ipmilan" class="stonith">
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-525400bbf613_last_0" operation_key="stonith-fence_ipmilan-525400bbf613_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="44:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;44:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="50" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="2" queue-time="0" op-digest="a1b3d562a7e722cbf686e1c2750888a3" op-secure-params=" passwd " op-secure-digest="2d0324390caa209c81283f07749d6dae"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="stonith-fence_ipmilan-525400b4f6bd" type="fence_ipmilan" class="stonith">
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-525400b4f6bd_last_0" operation_key="stonith-fence_ipmilan-525400b4f6bd_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="45:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;45:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="54" rc-code="7" op-status="0" interval="0" last-run="1493817971" last-rc-change="1493817971" exec-time="0" queue-time="0" op-digest="190a5d7184e8c0736ea8d50b25d1896b" op-secure-params=" passwd " op-secure-digest="e710ff47e4fa69f83f5f46f9de9570b6"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="redis_last_0" operation_key="redis_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="95:1:0:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:0;95:1:0:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="59" rc-code="0" op-status="0" interval="0" last-run="1493817972" last-rc-change="1493817972" exec-time="2559" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
60de42
+            <lrm_rsc_op id="redis_monitor_60000" operation_key="redis_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="80:2:0:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:0;80:2:0:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="61" rc-code="0" op-status="0" interval="60000" last-rc-change="1493817975" exec-time="121" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
60de42
+            <lrm_rsc_op id="redis_monitor_45000" operation_key="redis_monitor_45000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="79:2:0:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:0;79:2:0:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="62" rc-code="0" op-status="0" interval="45000" last-rc-change="1493817975" exec-time="117" queue-time="120" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
60de42
+          </lrm_resource>
60de42
+          <lrm_resource id="stonith-fence_ipmilan-5254005bdbb5" type="fence_ipmilan" class="stonith">
60de42
+            <lrm_rsc_op id="stonith-fence_ipmilan-5254005bdbb5_last_0" operation_key="stonith-fence_ipmilan-5254005bdbb5_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="46:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" transition-magic="0:7;46:1:7:e9fa94e7-ffc2-40ab-91b6-b69d0f610c13" on_node="controller-2" call-id="58" rc-code="7" op-status="0" interval="0" last-run="1493817972" last-rc-change="1493817972" exec-time="0" queue-time="1" op-digest="25087bde0e53bf4f4e4efb2b0bd9d0de" op-secure-params=" passwd " op-secure-digest="ecf31bd4bca13bb934be683ab04ee8cf"/>
60de42
+          </lrm_resource>
60de42
+        </lrm_resources>
60de42
+      </lrm>
60de42
+      <transient_attributes id="3">
60de42
+        <instance_attributes id="status-3">
60de42
+          <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
60de42
+          <nvpair id="status-3-master-redis" name="master-redis" value="1"/>
60de42
+        </instance_attributes>
60de42
+      </transient_attributes>
60de42
+    </node_state>
60de42
+    <node_state id="messaging-2" remote_node="true" uname="messaging-2" crm-debug-origin="post_cache_update" in_ccm="true">
60de42
+      <transient_attributes id="messaging-2">
60de42
+        <instance_attributes id="status-messaging-2">
60de42
+          <nvpair id="status-messaging-2-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@messaging-2"/>
60de42
+          <nvpair id="status-messaging-2-last-failure-rabbitmq" name="last-failure-rabbitmq" value="1493172992"/>
60de42
+        </instance_attributes>
60de42
+      </transient_attributes>
60de42
+      <lrm id="messaging-2">
60de42
+        <lrm_resources>
60de42
+          <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="rabbitmq_last_failure_0" operation_key="rabbitmq_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="7:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;7:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-2" call-id="1041497" rc-code="0" op-status="0" interval="0" last-run="1493817765" last-rc-change="1493817765" exec-time="3053" queue-time="0" op-digest="497e3cffa03bc3ed4830dc4b0ba109ae"/>
60de42
+            <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="7:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;7:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-2" call-id="1041497" rc-code="0" op-status="0" interval="0" last-run="1493817765" last-rc-change="1493817765" exec-time="3053" queue-time="0" op-digest="497e3cffa03bc3ed4830dc4b0ba109ae"/>
60de42
+            <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="37:18:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;37:18:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-2" call-id="1041515" rc-code="0" op-status="0" interval="10000" last-rc-change="1493817768" exec-time="3053" queue-time="0" op-digest="41cce5252b1fb189afe9efa00fb5e2cb"/>
60de42
+          </lrm_resource>
60de42
+        </lrm_resources>
60de42
+      </lrm>
60de42
+    </node_state>
60de42
+    <node_state id="messaging-0" remote_node="true" uname="messaging-0" in_ccm="true" crm-debug-origin="post_cache_update">
60de42
+      <transient_attributes id="messaging-0">
60de42
+        <instance_attributes id="status-messaging-0">
60de42
+          <nvpair id="status-messaging-0-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@messaging-0"/>
60de42
+          <nvpair id="status-messaging-0-last-failure-rabbitmq" name="last-failure-rabbitmq" value="1493330172"/>
60de42
+        </instance_attributes>
60de42
+      </transient_attributes>
60de42
+      <lrm id="messaging-0">
60de42
+        <lrm_resources>
60de42
+          <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="rabbitmq_last_failure_0" operation_key="rabbitmq_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="5:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;5:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-1" call-id="875374" rc-code="0" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="3026" queue-time="0" op-digest="497e3cffa03bc3ed4830dc4b0ba109ae"/>
60de42
+            <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="5:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;5:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-1" call-id="875374" rc-code="0" op-status="0" interval="0" last-run="1493817764" last-rc-change="1493817764" exec-time="3026" queue-time="0" op-digest="497e3cffa03bc3ed4830dc4b0ba109ae"/>
60de42
+            <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="40:18:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;40:18:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-1" call-id="875393" rc-code="0" op-status="0" interval="10000" last-rc-change="1493817768" exec-time="2985" queue-time="0" op-digest="41cce5252b1fb189afe9efa00fb5e2cb"/>
60de42
+          </lrm_resource>
60de42
+        </lrm_resources>
60de42
+      </lrm>
60de42
+    </node_state>
60de42
+    <node_state id="messaging-1" remote_node="true" uname="messaging-1" in_ccm="true" crm-debug-origin="post_cache_update">
60de42
+      <transient_attributes id="messaging-1">
60de42
+        <instance_attributes id="status-messaging-1">
60de42
+          <nvpair id="status-messaging-1-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@messaging-1"/>
60de42
+          <nvpair id="status-messaging-1-last-failure-rabbitmq" name="last-failure-rabbitmq" value="1493336946"/>
60de42
+        </instance_attributes>
60de42
+      </transient_attributes>
60de42
+      <lrm id="messaging-1">
60de42
+        <lrm_resources>
60de42
+          <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat">
60de42
+            <lrm_rsc_op id="rabbitmq_last_failure_0" operation_key="rabbitmq_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="6:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;6:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-1" call-id="251547" rc-code="0" op-status="0" interval="0" last-run="1493817765" last-rc-change="1493817765" exec-time="3074" queue-time="0" op-digest="497e3cffa03bc3ed4830dc4b0ba109ae"/>
60de42
+            <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="6:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;6:17:7:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-1" call-id="251547" rc-code="0" op-status="0" interval="0" last-run="1493817765" last-rc-change="1493817765" exec-time="3074" queue-time="0" op-digest="497e3cffa03bc3ed4830dc4b0ba109ae"/>
60de42
+            <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="43:18:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" transition-magic="0:0;43:18:0:e47e0f5b-bac4-432e-9993-f38bc43128ea" on_node="controller-1" call-id="251566" rc-code="0" op-status="0" interval="10000" last-rc-change="1493817768" exec-time="3025" queue-time="0" op-digest="41cce5252b1fb189afe9efa00fb5e2cb"/>
60de42
+          </lrm_resource>
60de42
+        </lrm_resources>
60de42
+      </lrm>
60de42
+    </node_state>
60de42
+  </status>
60de42
+</cib>
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From b08ff7bea963472a35e2b0fbb0a91e2a387b90a5 Mon Sep 17 00:00:00 2001
60de42
From: Andrew Beekhof <andrew@beekhof.net>
60de42
Date: Fri, 19 May 2017 17:44:37 +1000
60de42
Subject: [PATCH 03/13] Fix: PE: Prevent use-of-NULL when expanding containers
60de42
60de42
---
60de42
 lib/pengine/utils.c | 4 ++++
60de42
 1 file changed, 4 insertions(+)
60de42
60de42
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
60de42
index ff142f0..9e47f03 100644
60de42
--- a/lib/pengine/utils.c
60de42
+++ b/lib/pengine/utils.c
60de42
@@ -1690,6 +1690,10 @@ bool fix_remote_addr(resource_t * rsc)
60de42
         "pacemaker"
60de42
     };
60de42
 
60de42
+    if(rsc == NULL) {
60de42
+        return FALSE;
60de42
+    }
60de42
+
60de42
     name = "addr";
60de42
     value = g_hash_table_lookup(rsc->parameters, name);
60de42
     if (safe_str_eq(value, "#uname") == FALSE) {
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From 2d6b90ff01f91ad284b2ea5988cd3ccdec21aac3 Mon Sep 17 00:00:00 2001
60de42
From: Ken Gaillot <kgaillot@redhat.com>
60de42
Date: Fri, 19 May 2017 10:17:37 -0500
60de42
Subject: [PATCH 04/13] Low: pengine: improve detection of invalid constraints
60de42
60de42
The previous test was overbroad. The new test specifically checks
60de42
whether the constraint is between a resource running on a guest node
60de42
and the resource creating that guest node.
60de42
---
60de42
 pengine/graph.c | 17 +++++------------
60de42
 1 file changed, 5 insertions(+), 12 deletions(-)
60de42
60de42
diff --git a/pengine/graph.c b/pengine/graph.c
60de42
index 4e8c6dc..72df371 100644
60de42
--- a/pengine/graph.c
60de42
+++ b/pengine/graph.c
60de42
@@ -1311,20 +1311,13 @@ check_dump_input(int last_action, action_t * action, action_wrapper_t * wrapper)
60de42
     type &= ~pe_order_implies_then_printed;
60de42
     type &= ~pe_order_optional;
60de42
 
60de42
-    if (wrapper->action->node
60de42
+    if (is_not_set(type, pe_order_preserve)
60de42
         && action->rsc && action->rsc->fillers
60de42
-        && is_not_set(type, pe_order_preserve)
60de42
+        && wrapper->action->rsc && wrapper->action->node
60de42
         && wrapper->action->node->details->remote_rsc
60de42
-        && uber_parent(action->rsc) != uber_parent(wrapper->action->rsc)
60de42
-        ) {
60de42
-        /* This prevents user-defined ordering constraints between
60de42
-         * resources in remote nodes and the resources that
60de42
-         * define/represent a remote node.
60de42
-         *
60de42
-         * There is no known valid reason to allow this sort of thing
60de42
-         * but if one arises, we'd need to change the
60de42
-         * action->rsc->fillers clause to be more specific, possibly
60de42
-         * to check that it contained wrapper->action->rsc
60de42
+        && (wrapper->action->node->details->remote_rsc->container == action->rsc)) {
60de42
+        /* This prevents user-defined ordering constraints between resources
60de42
+         * running in a guest node and the resource that defines that node.
60de42
          */
60de42
         crm_warn("Invalid ordering constraint between %s and %s",
60de42
                  wrapper->action->rsc->id, action->rsc->id);
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From c9dcc29b1e9b77c87c6f6a653969f3f7ecb81e91 Mon Sep 17 00:00:00 2001
60de42
From: Ken Gaillot <kgaillot@redhat.com>
60de42
Date: Fri, 19 May 2017 18:28:48 -0500
60de42
Subject: [PATCH 05/13] Low: libpengine: don't double score when adding first
60de42
 allowed node
60de42
60de42
Not usually a problem in practice because first score added in a
60de42
symmetric cluster is 0
60de42
---
60de42
 lib/pengine/utils.c | 1 -
60de42
 1 file changed, 1 deletion(-)
60de42
60de42
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
60de42
index 9e47f03..16687e0 100644
60de42
--- a/lib/pengine/utils.c
60de42
+++ b/lib/pengine/utils.c
60de42
@@ -1299,7 +1299,6 @@ resource_node_score(resource_t * rsc, node_t * node, int score, const char *tag)
60de42
     match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
60de42
     if (match == NULL) {
60de42
         match = node_copy(node);
60de42
-        match->weight = merge_weights(score, node->weight);
60de42
         g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
60de42
     }
60de42
     match->weight = merge_weights(match->weight, score);
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From 74b95e26b815ab04609f05d02feb749a7dfed406 Mon Sep 17 00:00:00 2001
60de42
From: Ken Gaillot <kgaillot@redhat.com>
60de42
Date: Fri, 19 May 2017 18:38:27 -0500
60de42
Subject: [PATCH 06/13] Test: pengine: update regression tests for score change
60de42
60de42
---
60de42
 pengine/test10/remote-orphaned.scores   | 2 +-
60de42
 pengine/test10/whitebox-orphaned.scores | 2 +-
60de42
 2 files changed, 2 insertions(+), 2 deletions(-)
60de42
60de42
diff --git a/pengine/test10/remote-orphaned.scores b/pengine/test10/remote-orphaned.scores
60de42
index 53b4ada..c6f0957 100644
60de42
--- a/pengine/test10/remote-orphaned.scores
60de42
+++ b/pengine/test10/remote-orphaned.scores
60de42
@@ -15,7 +15,7 @@ clone_color: ping-1:1 allocation score on 18node3: 1
60de42
 clone_color: ping-1:2 allocation score on 18node1: 0
60de42
 clone_color: ping-1:2 allocation score on 18node2: 0
60de42
 clone_color: ping-1:2 allocation score on 18node3: 0
60de42
-clone_color: ping-1:2 allocation score on remote1: 2
60de42
+clone_color: ping-1:2 allocation score on remote1: 1
60de42
 clone_color: stateful-1:0 allocation score on 18node1: 11
60de42
 clone_color: stateful-1:0 allocation score on 18node2: -INFINITY
60de42
 clone_color: stateful-1:0 allocation score on 18node3: 0
60de42
diff --git a/pengine/test10/whitebox-orphaned.scores b/pengine/test10/whitebox-orphaned.scores
60de42
index 7f303b8..d626df2 100644
60de42
--- a/pengine/test10/whitebox-orphaned.scores
60de42
+++ b/pengine/test10/whitebox-orphaned.scores
60de42
@@ -26,7 +26,7 @@ native_color: A allocation score on lxc2: 0
60de42
 native_color: B allocation score on 18node1: 0
60de42
 native_color: B allocation score on 18node2: 0
60de42
 native_color: B allocation score on 18node3: 0
60de42
-native_color: B allocation score on lxc1: 200
60de42
+native_color: B allocation score on lxc1: 100
60de42
 native_color: B allocation score on lxc2: 0
60de42
 native_color: C allocation score on 18node1: 0
60de42
 native_color: C allocation score on 18node2: 0
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From 048c3e28d2dead3180e6cedca05460728e271d82 Mon Sep 17 00:00:00 2001
60de42
From: Ken Gaillot <kgaillot@redhat.com>
60de42
Date: Fri, 19 May 2017 16:51:47 -0500
60de42
Subject: [PATCH 07/13] Low: pengine: avoid double include
60de42
60de42
---
60de42
 pengine/container.c | 1 -
60de42
 1 file changed, 1 deletion(-)
60de42
60de42
diff --git a/pengine/container.c b/pengine/container.c
60de42
index 49b288a..bfef976 100644
60de42
--- a/pengine/container.c
60de42
+++ b/pengine/container.c
60de42
@@ -22,7 +22,6 @@
60de42
 #include <allocate.h>
60de42
 #include <notif.h>
60de42
 #include <utils.h>
60de42
-#include <allocate.h>
60de42
 
60de42
 #define VARIANT_CONTAINER 1
60de42
 #include <lib/pengine/variant.h>
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From ce01a26b779442e030a3f1aa6b1ce7cf46c925d0 Mon Sep 17 00:00:00 2001
60de42
From: Ken Gaillot <kgaillot@redhat.com>
60de42
Date: Tue, 9 May 2017 17:36:29 -0500
60de42
Subject: [PATCH 08/13] Fix: pengine: avoid use-of-NULL in bundle without
60de42
 primitive
60de42
60de42
---
60de42
 pengine/container.c | 4 +++-
60de42
 1 file changed, 3 insertions(+), 1 deletion(-)
60de42
60de42
diff --git a/pengine/container.c b/pengine/container.c
60de42
index bfef976..b63dcb3 100644
60de42
--- a/pengine/container.c
60de42
+++ b/pengine/container.c
60de42
@@ -262,7 +262,9 @@ container_expand(resource_t * rsc, pe_working_set_t * data_set)
60de42
 
60de42
 
60de42
         CRM_ASSERT(tuple);
60de42
-        if(fix_remote_addr(tuple->remote) && tuple->docker->allocated_to) {
60de42
+        if (tuple->docker && tuple->remote && tuple->docker->allocated_to
60de42
+            && fix_remote_addr(tuple->remote)) {
60de42
+
60de42
             // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
60de42
             xmlNode *nvpair = get_xpath_object("//nvpair[@name='addr']", tuple->remote->xml, LOG_ERR);
60de42
 
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From 25d6b6f0112c42e20d59305169b68472870a0b43 Mon Sep 17 00:00:00 2001
60de42
From: Ken Gaillot <kgaillot@redhat.com>
60de42
Date: Tue, 9 May 2017 17:48:51 -0500
60de42
Subject: [PATCH 09/13] Low: pengine,libpengine: avoid potential use-of-NULL
60de42
60de42
no risk in current code, but handles any future misuse
60de42
---
60de42
 lib/pengine/utils.c | 5 ++++-
60de42
 pengine/allocate.c  | 3 ++-
60de42
 2 files changed, 6 insertions(+), 2 deletions(-)
60de42
60de42
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
60de42
index 16687e0..3d4e60d 100644
60de42
--- a/lib/pengine/utils.c
60de42
+++ b/lib/pengine/utils.c
60de42
@@ -1699,7 +1699,7 @@ bool fix_remote_addr(resource_t * rsc)
60de42
         return FALSE;
60de42
     }
60de42
 
60de42
-    for (int lpc = 0; rsc && lpc < DIMOF(attr_list); lpc++) {
60de42
+    for (int lpc = 0; lpc < DIMOF(attr_list); lpc++) {
60de42
         name = attr_list[lpc];
60de42
         value = crm_element_value(rsc->xml, attr_list[lpc]);
60de42
         if (safe_str_eq(value, value_list[lpc]) == FALSE) {
60de42
@@ -1734,12 +1734,15 @@ rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node,
60de42
     const char *restart_list;
60de42
     const char *op_version;
60de42
 
60de42
+    CRM_ASSERT(node != NULL);
60de42
+
60de42
     data = g_hash_table_lookup(node->details->digest_cache, op_id);
60de42
     if (data) {
60de42
         return data;
60de42
     }
60de42
 
60de42
     data = calloc(1, sizeof(op_digest_cache_t));
60de42
+    CRM_ASSERT(data != NULL);
60de42
 
60de42
     digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
60de42
     digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
60de42
diff --git a/pengine/allocate.c b/pengine/allocate.c
60de42
index 3819c24..6b0025b 100644
60de42
--- a/pengine/allocate.c
60de42
+++ b/pengine/allocate.c
60de42
@@ -276,7 +276,8 @@ check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op
60de42
         key = NULL;
60de42
     }
60de42
 
60de42
-    crm_trace("Testing %s_%s_%d on %s", rsc->id, task, interval, active_node?active_node->details->uname:"N/A");
60de42
+    crm_trace("Testing %s_%s_%d on %s",
60de42
+              rsc->id, task, interval, active_node->details->uname);
60de42
     if (interval == 0 && safe_str_eq(task, RSC_STATUS)) {
60de42
         /* Reload based on the start action not a probe */
60de42
         task = RSC_START;
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From 32477db962b9798a323c7da27cece111519bf534 Mon Sep 17 00:00:00 2001
60de42
From: Ken Gaillot <kgaillot@redhat.com>
60de42
Date: Fri, 12 May 2017 17:54:04 -0500
60de42
Subject: [PATCH 10/13] Low: libpe_status: set #kind to "container" for bundle
60de42
 nodes
60de42
60de42
---
60de42
 lib/pengine/container.c | 29 +++++++++++++++++++++++------
60de42
 1 file changed, 23 insertions(+), 6 deletions(-)
60de42
60de42
diff --git a/lib/pengine/container.c b/lib/pengine/container.c
60de42
index 043f129..f93a60f 100644
60de42
--- a/lib/pengine/container.c
60de42
+++ b/lib/pengine/container.c
60de42
@@ -384,14 +384,13 @@ create_remote_resource(
60de42
 
60de42
         create_nvp(xml_obj, XML_OP_ATTR_ALLOW_MIGRATE, "false");
60de42
 
60de42
-        // Sets up node->details->remote_rsc->container == tuple->docker
60de42
+        /* This sets tuple->docker as tuple->remote's container, which is
60de42
+         * similar to what happens with guest nodes. This is how the PE knows
60de42
+         * that the bundle node is fenced by recovering docker, and that
60de42
+         * remote should be ordered relative to docker.
60de42
+         */
60de42
         create_nvp(xml_obj, XML_RSC_ATTR_CONTAINER, tuple->docker->id);
60de42
 
60de42
-        // TODO: Do this generically, eg with rsc->flags
60de42
-        // create_nvp(xml_obj, XML_RSC_ATTR_INTERNAL_RSC, "true"); // Suppress printing
60de42
-
60de42
-        // tuple->docker->fillers = g_list_append(tuple->docker->fillers, child);
60de42
-
60de42
         /* Ensure a node has been created for the guest (it may have already
60de42
          * been, if it has a permanent node attribute), and ensure its weight is
60de42
          * -INFINITY so no other resources can run on it.
60de42
@@ -422,6 +421,24 @@ create_remote_resource(
60de42
         }
60de42
 
60de42
         tuple->node->details->remote_rsc = tuple->remote;
60de42
+
60de42
+        /* #kind is irrelevant to bundles since it is only used in location
60de42
+         * constraint rules, and those don't matter for resources inside
60de42
+         * bundles. But just for clarity, a bundle is closer to "container"
60de42
+         * (guest node) than the "remote" set by pe_create_node().
60de42
+         */
60de42
+        g_hash_table_insert(tuple->node->details->attrs,
60de42
+                            strdup("#kind"), strdup("container"));
60de42
+
60de42
+        /* One effect of this is that setup_container() will add
60de42
+         * tuple->remote to tuple->docker's fillers, which will make
60de42
+         * rsc_contains_remote_node() true for tuple->docker.
60de42
+         *
60de42
+         * tuple->child does NOT get added to tuple->docker's fillers.
60de42
+         * The only noticeable effect if it did would be for its fail count to
60de42
+         * be taken into account when checking tuple->docker's migration
60de42
+         * threshold.
60de42
+         */
60de42
         parent->children = g_list_append(parent->children, tuple->remote);
60de42
     }
60de42
     return TRUE;
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From bd7f2b64841514d08fb5bbd4b990857e4ccd8def Mon Sep 17 00:00:00 2001
60de42
From: Ken Gaillot <kgaillot@redhat.com>
60de42
Date: Mon, 15 May 2017 16:11:45 -0500
60de42
Subject: [PATCH 11/13] Log: pengine: re-phrase trace log for internal
60de42
 constraints
60de42
60de42
This improves comments and one trace message, and involves slight
60de42
refactoring that I initially planned to do more with, but decided against.
60de42
---
60de42
 pengine/native.c | 35 ++++++++++++++++++++++++++---------
60de42
 1 file changed, 26 insertions(+), 9 deletions(-)
60de42
60de42
diff --git a/pengine/native.c b/pengine/native.c
60de42
index f30bc8b..82b73bb 100644
60de42
--- a/pengine/native.c
60de42
+++ b/pengine/native.c
60de42
@@ -1447,16 +1447,24 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
60de42
     if (rsc->container) {
60de42
         resource_t *remote_rsc = NULL;
60de42
 
60de42
-        /* find out if the container is associated with remote node connection resource */
60de42
+        /* A user can specify that a resource must start on a Pacemaker Remote
60de42
+         * node by explicitly configuring it with the container=NODENAME
60de42
+         * meta-attribute. This is of questionable merit, since location
60de42
+         * constraints can accomplish the same thing. But we support it, so here
60de42
+         * we check whether a resource (that is not itself a remote connection)
60de42
+         * has container set to a remote node or guest node resource.
60de42
+         */
60de42
         if (rsc->container->is_remote_node) {
60de42
             remote_rsc = rsc->container;
60de42
         } else if (rsc->is_remote_node == FALSE) {
60de42
             remote_rsc = rsc_contains_remote_node(data_set, rsc->container);
60de42
         }
60de42
 
60de42
-        /* if the container is a remote-node, force the resource within the container
60de42
-         * instead of colocating the resource with the container. */
60de42
         if (remote_rsc) {
60de42
+            /* The container represents a Pacemaker Remote node, so force the
60de42
+             * resource on the Pacemaker Remote node instead of colocating the
60de42
+             * resource with the container resource.
60de42
+             */
60de42
             GHashTableIter iter;
60de42
             node_t *node = NULL;
60de42
             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
60de42
@@ -1466,8 +1474,15 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
60de42
                 }
60de42
             }
60de42
         } else {
60de42
-            int score = 10000; /* Highly preferred but not essential */
60de42
-            crm_trace("Generating order and colocation rules for rsc %s with container %s", rsc->id, rsc->container->id);
60de42
+            /* This resource is either a filler for a container that does NOT
60de42
+             * represent a Pacemaker Remote node, or a Pacemaker Remote
60de42
+             * connection resource for a guest node or bundle.
60de42
+             */
60de42
+            int score;
60de42
+
60de42
+            crm_trace("Order and colocate %s relative to its container %s",
60de42
+                      rsc->id, rsc->container->id);
60de42
+
60de42
             custom_action_order(rsc->container, generate_op_key(rsc->container->id, RSC_START, 0), NULL,
60de42
                                 rsc, generate_op_key(rsc->id, RSC_START, 0), NULL,
60de42
                                 pe_order_implies_then | pe_order_runnable_left, data_set);
60de42
@@ -1476,11 +1491,13 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
60de42
                                 rsc->container, generate_op_key(rsc->container->id, RSC_STOP, 0), NULL,
60de42
                                 pe_order_implies_first, data_set);
60de42
 
60de42
-            if(is_not_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
60de42
-                score = INFINITY; /* Force them to run on the same host */ 
60de42
+            if (is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
60de42
+                score = 10000;    /* Highly preferred but not essential */
60de42
+            } else {
60de42
+                score = INFINITY; /* Force them to run on the same host */
60de42
             }
60de42
-            rsc_colocation_new("resource-with-container", NULL, score, rsc, rsc->container, NULL,
60de42
-                               NULL, data_set);
60de42
+            rsc_colocation_new("resource-with-container", NULL, score, rsc,
60de42
+                               rsc->container, NULL, NULL, data_set);
60de42
         }
60de42
     }
60de42
 
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From 56a7985a768cd42d0cfcb7b9fcea043652ed0aa0 Mon Sep 17 00:00:00 2001
60de42
From: Ken Gaillot <kgaillot@redhat.com>
60de42
Date: Mon, 15 May 2017 17:53:07 -0500
60de42
Subject: [PATCH 12/13] Low: pengine: improve internal bundle constraints
60de42
60de42
Ordering IP with docker requires pe_order_preserve.
60de42
60de42
Explicit ordering of remote relative to docker is redundant since it's handled
60de42
by native internal constraints.
60de42
60de42
Explicit colocation of remote with IP is both redundant (since both remote and
60de42
IP are colocated with docker) and harmful if mandatory (since it prohibits
60de42
running bundles on Pacemaker Remote nodes).
60de42
---
60de42
 pengine/container.c | 24 +++++++++---------------
60de42
 1 file changed, 9 insertions(+), 15 deletions(-)
60de42
60de42
diff --git a/pengine/container.c b/pengine/container.c
60de42
index b63dcb3..5bde89d 100644
60de42
--- a/pengine/container.c
60de42
+++ b/pengine/container.c
60de42
@@ -167,8 +167,10 @@ container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
60de42
             tuple->ip->cmds->internal_constraints(tuple->ip, data_set);
60de42
 
60de42
             // Start ip then docker
60de42
-            new_rsc_order(tuple->ip, RSC_START, tuple->docker, RSC_START, pe_order_runnable_left, data_set);
60de42
-            new_rsc_order(tuple->docker, RSC_STOP, tuple->ip, RSC_STOP, pe_order_implies_first, data_set);
60de42
+            new_rsc_order(tuple->ip, RSC_START, tuple->docker, RSC_START,
60de42
+                          pe_order_runnable_left|pe_order_preserve, data_set);
60de42
+            new_rsc_order(tuple->docker, RSC_STOP, tuple->ip, RSC_STOP,
60de42
+                          pe_order_implies_first|pe_order_preserve, data_set);
60de42
 
60de42
             id = crm_strdup_printf("%s-ip-with-docker-%d", rsc->id, tuple->offset);
60de42
             rsc_colocation_new(id, NULL, INFINITY, tuple->ip, tuple->docker, NULL, NULL, data_set);
60de42
@@ -176,20 +178,12 @@ container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
60de42
         }
60de42
 
60de42
         if(tuple->remote) {
60de42
+            /* This handles ordering and colocating remote relative to docker
60de42
+             * (via "resource-with-container"). Since IP is also ordered and
60de42
+             * colocated relative to docker, we don't need to do anything
60de42
+             * explicit here with IP.
60de42
+             */
60de42
             tuple->remote->cmds->internal_constraints(tuple->remote, data_set);
60de42
-            // Start docker then remote
60de42
-            new_rsc_order(
60de42
-                tuple->docker, RSC_START, tuple->remote, RSC_START, pe_order_runnable_left, data_set);
60de42
-            new_rsc_order(
60de42
-                tuple->remote, RSC_STOP, tuple->docker, RSC_STOP, pe_order_implies_first, data_set);
60de42
-
60de42
-            if(tuple->ip) {
60de42
-                id = crm_strdup_printf("%s-remote-with-ip-%d", rsc->id, tuple->offset);
60de42
-                rsc_colocation_new(id, NULL, INFINITY, tuple->remote, tuple->ip, NULL, NULL, data_set);
60de42
-                free(id);
60de42
-//            } else {
60de42
-                // remote-with-docker is already handled in native_internal_constraints() by 'resource-with-container'
60de42
-            }
60de42
         }
60de42
 
60de42
         if(tuple->child) {
60de42
-- 
60de42
1.8.3.1
60de42
60de42
60de42
From 6f0468b80ac6ec88644ce1517b3668bfd96a51e3 Mon Sep 17 00:00:00 2001
60de42
From: Ken Gaillot <kgaillot@redhat.com>
60de42
Date: Thu, 11 May 2017 17:59:10 -0500
60de42
Subject: [PATCH 13/13] Fix: libpe_status: correctly flag remote connection
60de42
 resources
60de42
60de42
Previously, unpack_resources() would set is_remote_node (when appropriate)
60de42
after calling common_unpack(). However, this meant that other paths to
60de42
common_unpack() (such as for bundles) did not get this set correctly.
60de42
60de42
Now, is_remote_node is set in common_unpack() so all paths are correct.
60de42
It's also the logical place to do it, since it really is part of unpacking,
60de42
and saves an extra call to xml_contains_remote_node().
60de42
---
60de42
 lib/pengine/complex.c |  1 +
60de42
 lib/pengine/unpack.c  |  5 -----
60de42
 pengine/native.c      | 11 ++++++-----
60de42
 3 files changed, 7 insertions(+), 10 deletions(-)
60de42
60de42
diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
60de42
index 6246b08..c99b5af 100644
60de42
--- a/lib/pengine/complex.c
60de42
+++ b/lib/pengine/complex.c
60de42
@@ -542,6 +542,7 @@ common_unpack(xmlNode * xml_obj, resource_t ** rsc,
60de42
     }
60de42
 
60de42
     if (xml_contains_remote_node((*rsc)->xml)) {
60de42
+        (*rsc)->is_remote_node = TRUE;
60de42
         if (g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CONTAINER)) {
60de42
             container_remote_node = 1;
60de42
         } else {
60de42
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
60de42
index 6ec7daf..df9a133 100644
60de42
--- a/lib/pengine/unpack.c
60de42
+++ b/lib/pengine/unpack.c
60de42
@@ -789,10 +789,6 @@ unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set)
60de42
         crm_trace("Beginning unpack... <%s id=%s... >", crm_element_name(xml_obj), ID(xml_obj));
60de42
         if (common_unpack(xml_obj, &new_rsc, NULL, data_set)) {
60de42
             data_set->resources = g_list_append(data_set->resources, new_rsc);
60de42
-
60de42
-            if (xml_contains_remote_node(xml_obj)) {
60de42
-                new_rsc->is_remote_node = TRUE;
60de42
-            }
60de42
             print_resource(LOG_DEBUG_3, "Added ", new_rsc, FALSE);
60de42
 
60de42
         } else {
60de42
@@ -1673,7 +1669,6 @@ create_fake_resource(const char *rsc_id, xmlNode * rsc_entry, pe_working_set_t *
60de42
         node_t *node;
60de42
 
60de42
         crm_debug("Detected orphaned remote node %s", rsc_id);
60de42
-        rsc->is_remote_node = TRUE;
60de42
         node = pe_find_node(data_set->nodes, rsc_id);
60de42
         if (node == NULL) {
60de42
 	        node = pe_create_node(rsc_id, rsc_id, "remote", NULL, data_set);
60de42
diff --git a/pengine/native.c b/pengine/native.c
60de42
index 82b73bb..2c4e2a1 100644
60de42
--- a/pengine/native.c
60de42
+++ b/pengine/native.c
60de42
@@ -1507,11 +1507,12 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
60de42
         rsc_avoids_remote_nodes(rsc);
60de42
     }
60de42
 
60de42
-    /* If this rsc is a remote connection resource associated
60de42
-     * with a container ( which will most likely be a virtual guest )
60de42
-     * do not allow the container to live on any remote-nodes.
60de42
-     * remote-nodes managing nested remote-nodes should not be allowed. */
60de42
-    if (rsc->is_remote_node && rsc->container) {
60de42
+    /* If this is a guest node's implicit remote connection, do not allow the
60de42
+     * guest resource to live on a Pacemaker Remote node, to avoid nesting
60de42
+     * remotes. However, allow bundles to run on remote nodes.
60de42
+     */
60de42
+    if (rsc->is_remote_node && rsc->container
60de42
+        && is_not_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
60de42
         rsc_avoids_remote_nodes(rsc->container);
60de42
     }
60de42
 }
60de42
-- 
60de42
1.8.3.1
60de42